From 15d47b45e431826bbd238e13fccb7f92008f7e40 Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Wed, 19 Jun 2024 15:05:56 -0300 Subject: [PATCH 01/29] Modifying docker base image for linux in order to pick stage and python interpreter versions at runtime from user data env --- docker/base/Dockerfile | 22 ++++++++++++++++------ docker/base/start.sh | 11 +++++++++++ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index eef5b4a2e7f..851258026e4 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -97,13 +97,20 @@ RUN wget https://download.java.net/java/GA/jdk15.0.2/0d1cfde4252546c6931946de8db tar -xzv --strip-components=1 -f openjdk-15.0.2_linux-x64_bin.tar.gz --directory $JAVA_HOME && \ rm -rf openjdk*.tar.gz $JAVA_HOME/jmods $JAVA_HOME/lib/src.zip -# Install Python 3. +# Install Python 3.7 RUN curl -sS https://www.python.org/ftp/python/3.7.7/Python-3.7.7.tgz | tar -C /tmp -xzv && \ cd /tmp/Python-3.7.7 && \ ./configure --enable-optimizations --enable-loadable-sqlite-extensions && make altinstall && \ rm -rf /tmp/Python-3.7.7 RUN pip3.7 --no-cache-dir install pipenv==2022.8.5 +# Install Python3.11 +RUN curl -sS https://www.python.org/ftp/python/3.11.4/Python-3.11.4.tgz | tar -C /tmp -xzv && \ + cd /tmp/Python-3.11.4 && \ + ./configure --enable-optimizations --enable-loadable-sqlite-extensions && make altinstall && \ + rm -rf /tmp/Python-3.11.4 +RUN pip3.11 --no-cache-dir install pipenv==2022.8.5 + # Install Node.js COPY setup_19.x /data RUN bash setup_19.x && apt-get update -y && apt-get install -y nodejs @@ -131,7 +138,8 @@ ENV ROOT_DIR $INSTALL_DIRECTORY/clusterfuzz ENV UPDATE_WEB_TESTS True ENV PYTHONPATH $INSTALL_DIRECTORY/clusterfuzz/src ENV RUN_CMD "python3.7 $ROOT_DIR/src/python/bot/startup/run.py" -ENV DEPLOYMENT_ZIP "linux-3.zip" +ENV PYTHON_RUN_CMD = "$ROOT_DIR/src/python/bot/startup/run.py" +#ENV DEPLOYMENT_ZIP "linux-3.zip" # Passwordless sudo (needed for AFL launcher). RUN groupadd nopwsudo && \ @@ -148,9 +156,11 @@ ENV LANG en_US.UTF-8 ENV PYTHONIOENCODING UTF-8 COPY setup_common.sh setup_clusterfuzz.sh start_clusterfuzz.sh setup_mock_metadata.sh Pipfile Pipfile.lock start.sh /data/ -RUN cd /data && \ - # Make pip3.7 the default so that pipenv install --system works. - mv /usr/local/bin/pip3.7 /usr/local/bin/pip && \ - pipenv install --deploy --system +#Moving this to start.sh so we can choose interpreter version at runtime +RUN cd /data +#RUN cd /data && \ +# # Make pip3.7 the default so that pipenv install --system works. +# mv /usr/local/bin/pip${PYTHON_INTERPRETER_VERSION} /usr/local/bin/pip && \ +# pipenv install --deploy --system CMD ["bash", "-ex", "/data/start.sh"] diff --git a/docker/base/start.sh b/docker/base/start.sh index 64029102035..41d5cb3b0e3 100644 --- a/docker/base/start.sh +++ b/docker/base/start.sh @@ -13,6 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +DEPLOYMENT_ZIP="linux3.zip" +if [[$CLUSTERFUZZ_STAGE == "candidate" ]]; then + DEPLOYMENT_ZIP="linux3-candidate.zip" +fi +export DEPLOYMENT_ZIP +export RUN_CMD="python${PYTHON_INTERPRETER_VERSION} $PYTHON_RUN_CMD" + +#Assert pipenv install --system works +mv /usr/local/bin/pip${PYTHON_INTERPRETER_VERSION} /usr/local/bin/pip && \ + pipenv install --deploy --system + source /data/setup_common.sh source /data/setup_clusterfuzz.sh From 27e7510e68b66ac58223396a1b939abeeb838093 Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Thu, 20 Jun 2024 01:27:23 +0000 Subject: [PATCH 02/29] Editing package command to take stage as an argument --- butler.py | 2 ++ src/local/butler/package.py | 22 +++++++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/butler.py b/butler.py index e0d51de4207..8366db29cdc 100644 --- a/butler.py +++ b/butler.py @@ -259,6 +259,8 @@ def main(): 'package', help='Package clusterfuzz with a staging revision') parser_package.add_argument( '-p', '--platform', choices=['linux', 'macos', 'windows', 'all']) + parser_package.add_argument( + '-s', '--stage', choices=['prod', 'candidate']) parser_deploy = subparsers.add_parser('deploy', help='Deploy to Appengine') parser_deploy.add_argument( diff --git a/src/local/butler/package.py b/src/local/butler/package.py index 5824614aece..7df77fb9575 100644 --- a/src/local/butler/package.py +++ b/src/local/butler/package.py @@ -70,7 +70,9 @@ def _get_files(path): def package(revision, target_zip_dir=constants.PACKAGE_TARGET_ZIP_DIRECTORY, target_manifest_path=constants.PACKAGE_TARGET_MANIFEST_PATH, - platform_name=None): + platform_name=None, + stage='prod', + python3=False): """Prepare clusterfuzz-source.zip.""" is_ci = os.getenv('TEST_BOT_ENVIRONMENT') if not is_ci and common.is_git_dirty(): @@ -95,7 +97,17 @@ def package(revision, target_zip_name = constants.LEGACY_ZIP_NAME if platform_name: +<<<<<<< HEAD target_zip_name = platform_name + '-3.zip' +======= + if python3: + if stage=="prod": + target_zip_name = platform_name + '-3.zip' + else: + target_zip_name = platform_name + '-3-candidate.zip' + else: + target_zip_name = platform_name + '.zip' +>>>>>>> 5f3d71a3 (Editing package command to take stage as an argument) target_zip_path = os.path.join(target_zip_dir, target_zip_name) _clear_zip(target_zip_path) @@ -136,11 +148,15 @@ def package(revision, def execute(args): + stage = 'prod' + if args.stage == 'candidate': + stage = 'candidate' if args.platform == 'all': for platform_name in list(constants.PLATFORMS.keys()): package( revision=common.compute_staging_revision(), - platform_name=platform_name) + platform_name=platform_name, + stage=stage) else: package( - revision=common.compute_staging_revision(), platform_name=args.platform) + revision=common.compute_staging_revision(), platform_name=args.platform, stage=stage) From c4192a7f76fad987cc0966ae5bff2d48d95555e3 Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Thu, 20 Jun 2024 01:42:27 +0000 Subject: [PATCH 03/29] Adding stage argument to butler deploy --- butler.py | 2 ++ src/local/butler/deploy.py | 35 ++++++++++++++++++++++++++++------- src/local/butler/package.py | 4 ---- 3 files changed, 30 insertions(+), 11 deletions(-) diff --git a/butler.py b/butler.py index 8366db29cdc..9850291d464 100644 --- a/butler.py +++ b/butler.py @@ -276,6 +276,8 @@ def main(): '--prod', action='store_true', help='Deploy to production.') parser_deploy.add_argument( '--targets', nargs='*', default=['appengine', 'k8s', 'zips']) + parser_deploy.add_argument( + '--stage', '-s', choices=['prod', 'candidate']) parser_run_server = subparsers.add_parser( 'run_server', help='Run the local Clusterfuzz server.') diff --git a/src/local/butler/deploy.py b/src/local/butler/deploy.py index 32d14bbb6d6..315bba1c3f1 100644 --- a/src/local/butler/deploy.py +++ b/src/local/butler/deploy.py @@ -92,7 +92,8 @@ def _deploy_app_prod(project, yaml_paths, package_zip_paths, deploy_appengine=True, - test_deployment=False): + test_deployment=False, + stage='prod'): """Deploy app in production.""" if deploy_appengine: services = _get_services(yaml_paths) @@ -116,7 +117,8 @@ def _deploy_app_prod(project, _deploy_manifest( deployment_bucket, constants.PACKAGE_TARGET_MANIFEST_PATH, - test_deployment=test_deployment) + test_deployment=test_deployment, + stage=stage) def _deploy_app_staging(project, yaml_paths): @@ -227,8 +229,16 @@ def _deploy_zip(bucket_name, zip_path, test_deployment=False): os.path.basename(zip_path))) -def _deploy_manifest(bucket_name, manifest_path, test_deployment=False): +def _deploy_manifest(bucket_name, manifest_path, test_deployment=False, stage='prod'): """Deploy source manifest to GCS.""" + if sys.version_info.major == 3: + if stage=='prod': + manifest_suffix = '.3' + else: + manifest_suffix = '.3-candidate' + else: + manifest_suffix = '' + if test_deployment: common.execute(f'gsutil cp {manifest_path} ' f'gs://{bucket_name}/test-deployment/' @@ -390,7 +400,9 @@ def _prod_deployment_helper(config_dir, package_zip_paths, deploy_appengine=True, deploy_k8s=True, - test_deployment=False): + python3=True, + test_deployment=False, + stage='prod'): """Helper for production deployment.""" config = local_config.Config() deployment_bucket = config.get('project.deployment.bucket') @@ -418,7 +430,8 @@ def _prod_deployment_helper(config_dir, yaml_paths, package_zip_paths, deploy_appengine=deploy_appengine, - test_deployment=test_deployment) + test_deployment=test_deployment, + stage=stage) if deploy_appengine: common.execute( @@ -525,11 +538,17 @@ def execute(args): deploy_k8s = False deploy_zips = True + stage = args.stage + if args.stage == 'candidate': + stage='candidate' + + is_python3 = sys.version_info.major == 3 package_zip_paths = [] if deploy_zips: for platform_name in platforms: package_zip_paths.append( - package.package(revision, platform_name=platform_name)) + package.package( + revision, platform_name=platform_name, python3=is_python3, stage=stage)) else: # package.package calls these, so only set these up if we're not packaging, # since they can be fairly slow. @@ -553,7 +572,9 @@ def execute(args): package_zip_paths, deploy_appengine, deploy_k8s, - test_deployment=test_deployment) + python3=is_python3, + test_deployment=test_deployment, + stage=stage) with open(constants.PACKAGE_TARGET_MANIFEST_PATH) as f: print('Source updated to %s' % f.read()) diff --git a/src/local/butler/package.py b/src/local/butler/package.py index 7df77fb9575..de797bb0857 100644 --- a/src/local/butler/package.py +++ b/src/local/butler/package.py @@ -97,9 +97,6 @@ def package(revision, target_zip_name = constants.LEGACY_ZIP_NAME if platform_name: -<<<<<<< HEAD - target_zip_name = platform_name + '-3.zip' -======= if python3: if stage=="prod": target_zip_name = platform_name + '-3.zip' @@ -107,7 +104,6 @@ def package(revision, target_zip_name = platform_name + '-3-candidate.zip' else: target_zip_name = platform_name + '.zip' ->>>>>>> 5f3d71a3 (Editing package command to take stage as an argument) target_zip_path = os.path.join(target_zip_dir, target_zip_name) _clear_zip(target_zip_path) From 342b45307e1b34dbc5b63bd7b7fcfa75d52cdbc1 Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Thu, 20 Jun 2024 11:57:22 -0300 Subject: [PATCH 04/29] Adding stage awareness to update task --- .../_internal/bot/tasks/update_task.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/clusterfuzz/_internal/bot/tasks/update_task.py b/src/clusterfuzz/_internal/bot/tasks/update_task.py index 4b87840710d..1c5964fd476 100644 --- a/src/clusterfuzz/_internal/bot/tasks/update_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/update_task.py @@ -39,11 +39,17 @@ from clusterfuzz._internal.system import process_handler from clusterfuzz._internal.system import shell +CLUSTERFUZZ_STAGE = os.environ['CLUSTERFUZZ_STAGE'] + TESTS_LAST_UPDATE_KEY = 'tests_last_update' TESTS_UPDATE_INTERVAL_DAYS = 1 -MANIFEST_FILENAME = 'clusterfuzz-source.manifest.3' +REMOTE_MANIFEST_FILENAME = 'clusterfuzz-source.manifest' +if sys.version_info.major == 3: + REMOTE_MANIFEST_FILENAME += '.3' +if CLUSTERFUZZ_STAGE == 'candidate': + REMOTE_MANIFEST_FILENAME += '-candidate' def _rename_dll_for_update(absolute_filepath): """Rename a DLL to allow for updates.""" @@ -59,7 +65,12 @@ def _platform_deployment_filename(): 'Darwin': 'macos' } - base_filename = platform_mappings[platform.system()] + '-3' + base_filename = platform_mappings[platform.system()] + if sys.version_info.major == 3: + base_filename += '-3' + if CLUSTERFUZZ_STAGE == 'candidate': + base_filename += '-candidate' + return base_filename + '.zip' @@ -81,7 +92,7 @@ def get_source_url(): def get_source_manifest_url(): """Return the source manifest URL.""" - return _deployment_file_url(MANIFEST_FILENAME) + return _deployment_file_url(REMOTE_MANIFEST_FILENAME) def clear_old_files(directory, extracted_file_set): From 2010400778e205886a19ae728efe48b557b0f266 Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Thu, 20 Jun 2024 14:10:54 -0300 Subject: [PATCH 05/29] Updating remote tasks so the stage command works with the new package task signature --- src/local/remote/handlers/posix.py | 3 ++- src/local/remote/handlers/windows.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/local/remote/handlers/posix.py b/src/local/remote/handlers/posix.py index 6887b52c09c..e937a67c1db 100644 --- a/src/local/remote/handlers/posix.py +++ b/src/local/remote/handlers/posix.py @@ -130,7 +130,8 @@ def stage(self, config_dir): local_zip_path = package.package( revision=butler_common.compute_staging_revision(), - platform_name=self.platform) + platform_name=self.platform, + stage='prod') self._copy_staging_archive_from_local_to_remote(local_zip_path) self._run(f'cd {self.clusterfuzz_parent_path} && ' diff --git a/src/local/remote/handlers/windows.py b/src/local/remote/handlers/windows.py index 4fef18e258c..4e821db63a8 100644 --- a/src/local/remote/handlers/windows.py +++ b/src/local/remote/handlers/windows.py @@ -119,7 +119,8 @@ def stage(self, config_dir): zip_path = package.package( revision=butler_common.compute_staging_revision(), - platform_name='windows') + platform_name='windows', + stage='prod') remote_zip_path = ( f'{self.clusterfuzz_parent_path}\\{self.staging_source_filename}') api.put(zip_path, remote_zip_path) From 309d14d97eb8c4162203a2e9206d272b7c79a2a6 Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Mon, 8 Jul 2024 20:30:06 +0000 Subject: [PATCH 06/29] Using release instead of stage to define dual environments --- butler.py | 4 ++-- docker/base/Dockerfile | 8 +------- docker/base/start.sh | 2 +- src/local/butler/deploy.py | 26 +++++++++++++++----------- src/local/remote/handlers/posix.py | 2 +- src/local/remote/handlers/windows.py | 2 +- 6 files changed, 21 insertions(+), 23 deletions(-) diff --git a/butler.py b/butler.py index 9850291d464..169601a6d94 100644 --- a/butler.py +++ b/butler.py @@ -260,7 +260,7 @@ def main(): parser_package.add_argument( '-p', '--platform', choices=['linux', 'macos', 'windows', 'all']) parser_package.add_argument( - '-s', '--stage', choices=['prod', 'candidate']) + '-r', '--release', choices=['prod', 'candidate']) parser_deploy = subparsers.add_parser('deploy', help='Deploy to Appengine') parser_deploy.add_argument( @@ -277,7 +277,7 @@ def main(): parser_deploy.add_argument( '--targets', nargs='*', default=['appengine', 'k8s', 'zips']) parser_deploy.add_argument( - '--stage', '-s', choices=['prod', 'candidate']) + '--release', '-r', choices=['prod', 'candidate']) parser_run_server = subparsers.add_parser( 'run_server', help='Run the local Clusterfuzz server.') diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index 851258026e4..06d1704fe9b 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -138,7 +138,7 @@ ENV ROOT_DIR $INSTALL_DIRECTORY/clusterfuzz ENV UPDATE_WEB_TESTS True ENV PYTHONPATH $INSTALL_DIRECTORY/clusterfuzz/src ENV RUN_CMD "python3.7 $ROOT_DIR/src/python/bot/startup/run.py" -ENV PYTHON_RUN_CMD = "$ROOT_DIR/src/python/bot/startup/run.py" +ENV PYTHON_RUN_CMD "$ROOT_DIR/src/python/bot/startup/run.py" #ENV DEPLOYMENT_ZIP "linux-3.zip" # Passwordless sudo (needed for AFL launcher). @@ -156,11 +156,5 @@ ENV LANG en_US.UTF-8 ENV PYTHONIOENCODING UTF-8 COPY setup_common.sh setup_clusterfuzz.sh start_clusterfuzz.sh setup_mock_metadata.sh Pipfile Pipfile.lock start.sh /data/ -#Moving this to start.sh so we can choose interpreter version at runtime -RUN cd /data -#RUN cd /data && \ -# # Make pip3.7 the default so that pipenv install --system works. -# mv /usr/local/bin/pip${PYTHON_INTERPRETER_VERSION} /usr/local/bin/pip && \ -# pipenv install --deploy --system CMD ["bash", "-ex", "/data/start.sh"] diff --git a/docker/base/start.sh b/docker/base/start.sh index 41d5cb3b0e3..e73428c6c43 100644 --- a/docker/base/start.sh +++ b/docker/base/start.sh @@ -20,7 +20,7 @@ fi export DEPLOYMENT_ZIP export RUN_CMD="python${PYTHON_INTERPRETER_VERSION} $PYTHON_RUN_CMD" -#Assert pipenv install --system works +# Assert pipenv install --system works mv /usr/local/bin/pip${PYTHON_INTERPRETER_VERSION} /usr/local/bin/pip && \ pipenv install --deploy --system diff --git a/src/local/butler/deploy.py b/src/local/butler/deploy.py index 315bba1c3f1..3a857ffa76e 100644 --- a/src/local/butler/deploy.py +++ b/src/local/butler/deploy.py @@ -93,7 +93,7 @@ def _deploy_app_prod(project, package_zip_paths, deploy_appengine=True, test_deployment=False, - stage='prod'): + release='prod'): """Deploy app in production.""" if deploy_appengine: services = _get_services(yaml_paths) @@ -118,7 +118,7 @@ def _deploy_app_prod(project, deployment_bucket, constants.PACKAGE_TARGET_MANIFEST_PATH, test_deployment=test_deployment, - stage=stage) + release=release) def _deploy_app_staging(project, yaml_paths): @@ -229,16 +229,18 @@ def _deploy_zip(bucket_name, zip_path, test_deployment=False): os.path.basename(zip_path))) -def _deploy_manifest(bucket_name, manifest_path, test_deployment=False, stage='prod'): +def _deploy_manifest(bucket_name, manifest_path, test_deployment=False, release='prod'): """Deploy source manifest to GCS.""" if sys.version_info.major == 3: - if stage=='prod': + if release == 'prod': manifest_suffix = '.3' else: manifest_suffix = '.3-candidate' else: manifest_suffix = '' + manifest_path += manifest_suffix + if test_deployment: common.execute(f'gsutil cp {manifest_path} ' f'gs://{bucket_name}/test-deployment/' @@ -402,7 +404,7 @@ def _prod_deployment_helper(config_dir, deploy_k8s=True, python3=True, test_deployment=False, - stage='prod'): + release='prod'): """Helper for production deployment.""" config = local_config.Config() deployment_bucket = config.get('project.deployment.bucket') @@ -431,7 +433,7 @@ def _prod_deployment_helper(config_dir, package_zip_paths, deploy_appengine=deploy_appengine, test_deployment=test_deployment, - stage=stage) + release=release) if deploy_appengine: common.execute( @@ -538,9 +540,11 @@ def execute(args): deploy_k8s = False deploy_zips = True - stage = args.stage - if args.stage == 'candidate': - stage='candidate' + release = args.release + if args.release == 'candidate': + release='candidate' + else: + release = 'prod' is_python3 = sys.version_info.major == 3 package_zip_paths = [] @@ -548,7 +552,7 @@ def execute(args): for platform_name in platforms: package_zip_paths.append( package.package( - revision, platform_name=platform_name, python3=is_python3, stage=stage)) + revision, platform_name=platform_name, python3=is_python3, release=release)) else: # package.package calls these, so only set these up if we're not packaging, # since they can be fairly slow. @@ -574,7 +578,7 @@ def execute(args): deploy_k8s, python3=is_python3, test_deployment=test_deployment, - stage=stage) + release=release) with open(constants.PACKAGE_TARGET_MANIFEST_PATH) as f: print('Source updated to %s' % f.read()) diff --git a/src/local/remote/handlers/posix.py b/src/local/remote/handlers/posix.py index e937a67c1db..8da4341f254 100644 --- a/src/local/remote/handlers/posix.py +++ b/src/local/remote/handlers/posix.py @@ -131,7 +131,7 @@ def stage(self, config_dir): local_zip_path = package.package( revision=butler_common.compute_staging_revision(), platform_name=self.platform, - stage='prod') + release='prod') self._copy_staging_archive_from_local_to_remote(local_zip_path) self._run(f'cd {self.clusterfuzz_parent_path} && ' diff --git a/src/local/remote/handlers/windows.py b/src/local/remote/handlers/windows.py index 4e821db63a8..53e81221ce2 100644 --- a/src/local/remote/handlers/windows.py +++ b/src/local/remote/handlers/windows.py @@ -120,7 +120,7 @@ def stage(self, config_dir): zip_path = package.package( revision=butler_common.compute_staging_revision(), platform_name='windows', - stage='prod') + release='prod') remote_zip_path = ( f'{self.clusterfuzz_parent_path}\\{self.staging_source_filename}') api.put(zip_path, remote_zip_path) From e3734bbaf2661915dace5c8cdab6e2cb904982d2 Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Mon, 8 Jul 2024 20:33:37 +0000 Subject: [PATCH 07/29] Removing further references for stage --- src/local/butler/package.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/local/butler/package.py b/src/local/butler/package.py index de797bb0857..97ee7400bee 100644 --- a/src/local/butler/package.py +++ b/src/local/butler/package.py @@ -71,7 +71,7 @@ def package(revision, target_zip_dir=constants.PACKAGE_TARGET_ZIP_DIRECTORY, target_manifest_path=constants.PACKAGE_TARGET_MANIFEST_PATH, platform_name=None, - stage='prod', + release='prod', python3=False): """Prepare clusterfuzz-source.zip.""" is_ci = os.getenv('TEST_BOT_ENVIRONMENT') @@ -98,7 +98,7 @@ def package(revision, target_zip_name = constants.LEGACY_ZIP_NAME if platform_name: if python3: - if stage=="prod": + if release=="prod": target_zip_name = platform_name + '-3.zip' else: target_zip_name = platform_name + '-3-candidate.zip' @@ -144,15 +144,15 @@ def package(revision, def execute(args): - stage = 'prod' - if args.stage == 'candidate': - stage = 'candidate' + release = 'prod' + if args.release == 'candidate': + release = 'candidate' if args.platform == 'all': for platform_name in list(constants.PLATFORMS.keys()): package( revision=common.compute_staging_revision(), platform_name=platform_name, - stage=stage) + release=release) else: package( - revision=common.compute_staging_revision(), platform_name=args.platform, stage=stage) + revision=common.compute_staging_revision(), platform_name=args.platform, release=release) From 344cb34eb72afcc48fb854a2dabe173bb32ca59f Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Mon, 8 Jul 2024 20:38:26 +0000 Subject: [PATCH 08/29] Renaming CLUSTERFUZZ_STAGE to CLUSTERFUZZ_RELEASE in env vars --- docker/base/start.sh | 2 +- src/clusterfuzz/_internal/bot/tasks/update_task.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/base/start.sh b/docker/base/start.sh index e73428c6c43..512da992877 100644 --- a/docker/base/start.sh +++ b/docker/base/start.sh @@ -14,7 +14,7 @@ # limitations under the License. DEPLOYMENT_ZIP="linux3.zip" -if [[$CLUSTERFUZZ_STAGE == "candidate" ]]; then +if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then DEPLOYMENT_ZIP="linux3-candidate.zip" fi export DEPLOYMENT_ZIP diff --git a/src/clusterfuzz/_internal/bot/tasks/update_task.py b/src/clusterfuzz/_internal/bot/tasks/update_task.py index 1c5964fd476..52f0f620a8d 100644 --- a/src/clusterfuzz/_internal/bot/tasks/update_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/update_task.py @@ -39,7 +39,7 @@ from clusterfuzz._internal.system import process_handler from clusterfuzz._internal.system import shell -CLUSTERFUZZ_STAGE = os.environ['CLUSTERFUZZ_STAGE'] +CLUSTERFUZZ_RELEASE = os.getenv['CLUSTERFUZZ_STAGE'] TESTS_LAST_UPDATE_KEY = 'tests_last_update' TESTS_UPDATE_INTERVAL_DAYS = 1 @@ -48,7 +48,7 @@ REMOTE_MANIFEST_FILENAME = 'clusterfuzz-source.manifest' if sys.version_info.major == 3: REMOTE_MANIFEST_FILENAME += '.3' -if CLUSTERFUZZ_STAGE == 'candidate': +if CLUSTERFUZZ_RELEASE == 'candidate': REMOTE_MANIFEST_FILENAME += '-candidate' def _rename_dll_for_update(absolute_filepath): @@ -68,7 +68,7 @@ def _platform_deployment_filename(): base_filename = platform_mappings[platform.system()] if sys.version_info.major == 3: base_filename += '-3' - if CLUSTERFUZZ_STAGE == 'candidate': + if CLUSTERFUZZ_RELEASE == 'candidate': base_filename += '-candidate' return base_filename + '.zip' From 407bfa930c56b9bbe9591cb35a1b00f003ca8923 Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Mon, 8 Jul 2024 20:42:49 +0000 Subject: [PATCH 09/29] Sending manifest files to the correct place --- src/local/butler/deploy.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/local/butler/deploy.py b/src/local/butler/deploy.py index 3a857ffa76e..6af96323cd2 100644 --- a/src/local/butler/deploy.py +++ b/src/local/butler/deploy.py @@ -233,22 +233,22 @@ def _deploy_manifest(bucket_name, manifest_path, test_deployment=False, release= """Deploy source manifest to GCS.""" if sys.version_info.major == 3: if release == 'prod': - manifest_suffix = '.3' + manifest_suffix = '3' else: - manifest_suffix = '.3-candidate' + manifest_suffix = '3-candidate' else: manifest_suffix = '' - manifest_path += manifest_suffix + remote_manifest_path = f'clusterfuzz-source.manifest.{manifest_suffix}' if test_deployment: common.execute(f'gsutil cp {manifest_path} ' f'gs://{bucket_name}/test-deployment/' - f'clusterfuzz-source.manifest.3') + f'{remote_manifest_path}') else: common.execute(f'gsutil cp {manifest_path} ' f'gs://{bucket_name}/' - f'clusterfuzz-source.manifest.3') + f'{remote_manifest_path}') def _update_deployment_manager(project, name, config_path): From c3fa7343978a86c9d49367391166d84e68676994 Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Mon, 8 Jul 2024 20:57:02 +0000 Subject: [PATCH 10/29] Making python3 the default for butler package --- src/local/butler/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/local/butler/package.py b/src/local/butler/package.py index 97ee7400bee..a157f704145 100644 --- a/src/local/butler/package.py +++ b/src/local/butler/package.py @@ -72,7 +72,7 @@ def package(revision, target_manifest_path=constants.PACKAGE_TARGET_MANIFEST_PATH, platform_name=None, release='prod', - python3=False): + python3=True): """Prepare clusterfuzz-source.zip.""" is_ci = os.getenv('TEST_BOT_ENVIRONMENT') if not is_ci and common.is_git_dirty(): From f45071242edb5cefa9ef6b8fe309785a1c5f43d5 Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Mon, 8 Jul 2024 21:24:14 +0000 Subject: [PATCH 11/29] Fixing CLUSTERFUZZ_STAGE value in update task --- src/clusterfuzz/_internal/bot/tasks/update_task.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/clusterfuzz/_internal/bot/tasks/update_task.py b/src/clusterfuzz/_internal/bot/tasks/update_task.py index 52f0f620a8d..43c5d30cee5 100644 --- a/src/clusterfuzz/_internal/bot/tasks/update_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/update_task.py @@ -39,7 +39,7 @@ from clusterfuzz._internal.system import process_handler from clusterfuzz._internal.system import shell -CLUSTERFUZZ_RELEASE = os.getenv['CLUSTERFUZZ_STAGE'] +CLUSTERFUZZ_RELEASE = os.getenv['CLUSTERFUZZ_RELEASE'] TESTS_LAST_UPDATE_KEY = 'tests_last_update' TESTS_UPDATE_INTERVAL_DAYS = 1 From e1174b2827c6907fc14bfa0c4481e3344a86083c Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Wed, 10 Jul 2024 17:06:31 +0000 Subject: [PATCH 12/29] Removing interpreter choice at runtime for clusterfuzz startup --- docker/base/Dockerfile | 15 +++++---------- docker/base/start.sh | 5 ----- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index 06d1704fe9b..b9bac2e120b 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -104,13 +104,6 @@ RUN curl -sS https://www.python.org/ftp/python/3.7.7/Python-3.7.7.tgz | tar -C / rm -rf /tmp/Python-3.7.7 RUN pip3.7 --no-cache-dir install pipenv==2022.8.5 -# Install Python3.11 -RUN curl -sS https://www.python.org/ftp/python/3.11.4/Python-3.11.4.tgz | tar -C /tmp -xzv && \ - cd /tmp/Python-3.11.4 && \ - ./configure --enable-optimizations --enable-loadable-sqlite-extensions && make altinstall && \ - rm -rf /tmp/Python-3.11.4 -RUN pip3.11 --no-cache-dir install pipenv==2022.8.5 - # Install Node.js COPY setup_19.x /data RUN bash setup_19.x && apt-get update -y && apt-get install -y nodejs @@ -138,8 +131,6 @@ ENV ROOT_DIR $INSTALL_DIRECTORY/clusterfuzz ENV UPDATE_WEB_TESTS True ENV PYTHONPATH $INSTALL_DIRECTORY/clusterfuzz/src ENV RUN_CMD "python3.7 $ROOT_DIR/src/python/bot/startup/run.py" -ENV PYTHON_RUN_CMD "$ROOT_DIR/src/python/bot/startup/run.py" -#ENV DEPLOYMENT_ZIP "linux-3.zip" # Passwordless sudo (needed for AFL launcher). RUN groupadd nopwsudo && \ @@ -156,5 +147,9 @@ ENV LANG en_US.UTF-8 ENV PYTHONIOENCODING UTF-8 COPY setup_common.sh setup_clusterfuzz.sh start_clusterfuzz.sh setup_mock_metadata.sh Pipfile Pipfile.lock start.sh /data/ - +RUN cd /data && \ + # Make pip3.7 the default so that pipenv install --system works. + mv /usr/local/bin/pip3.7 /usr/local/bin/pip && \ + pipenv install --deploy --system + CMD ["bash", "-ex", "/data/start.sh"] diff --git a/docker/base/start.sh b/docker/base/start.sh index 512da992877..e0943f6f68d 100644 --- a/docker/base/start.sh +++ b/docker/base/start.sh @@ -18,11 +18,6 @@ if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then DEPLOYMENT_ZIP="linux3-candidate.zip" fi export DEPLOYMENT_ZIP -export RUN_CMD="python${PYTHON_INTERPRETER_VERSION} $PYTHON_RUN_CMD" - -# Assert pipenv install --system works -mv /usr/local/bin/pip${PYTHON_INTERPRETER_VERSION} /usr/local/bin/pip && \ - pipenv install --deploy --system source /data/setup_common.sh source /data/setup_clusterfuzz.sh From 6ea723ae14f6964be5841748fdf42f15264c2822 Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Wed, 10 Jul 2024 17:27:38 +0000 Subject: [PATCH 13/29] Default CLUSTERFUZZ_STAGE to prod so we do not break environments that do not implement prog rollout --- src/clusterfuzz/_internal/bot/tasks/update_task.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/clusterfuzz/_internal/bot/tasks/update_task.py b/src/clusterfuzz/_internal/bot/tasks/update_task.py index 43c5d30cee5..0992c35a8fa 100644 --- a/src/clusterfuzz/_internal/bot/tasks/update_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/update_task.py @@ -40,6 +40,8 @@ from clusterfuzz._internal.system import shell CLUSTERFUZZ_RELEASE = os.getenv['CLUSTERFUZZ_RELEASE'] +if CLUSTERFUZZ_RELEASE == None: + CLUSTERFUZZ_RELEASE = 'prod' TESTS_LAST_UPDATE_KEY = 'tests_last_update' TESTS_UPDATE_INTERVAL_DAYS = 1 From 82666ab7d7d32c49ddab439f0d007c7ab6dcaf3d Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Wed, 10 Jul 2024 17:32:20 +0000 Subject: [PATCH 14/29] Adding release and docker image to loggin --- src/clusterfuzz/_internal/metrics/logs.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/clusterfuzz/_internal/metrics/logs.py b/src/clusterfuzz/_internal/metrics/logs.py index 7e3b92a4ab5..bf4ad9ee7d4 100644 --- a/src/clusterfuzz/_internal/metrics/logs.py +++ b/src/clusterfuzz/_internal/metrics/logs.py @@ -553,7 +553,9 @@ def emit(level, message, exc_info=None, **extras): 'path': path_name, 'line': line_number, 'method': method_name - } + }, + 'release': os.environ.get('CLUSTERFUZZ_STAGE', 'prod'), + 'docker_image': os.environ.get('IMAGE', '') }) From 9e538e225acac043b32fe9bc41bf4b72305cdc01 Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Wed, 10 Jul 2024 18:33:46 +0000 Subject: [PATCH 15/29] Code sanitization --- src/local/butler/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/local/butler/package.py b/src/local/butler/package.py index a157f704145..c4c963369b7 100644 --- a/src/local/butler/package.py +++ b/src/local/butler/package.py @@ -98,7 +98,7 @@ def package(revision, target_zip_name = constants.LEGACY_ZIP_NAME if platform_name: if python3: - if release=="prod": + if release == 'prod': target_zip_name = platform_name + '-3.zip' else: target_zip_name = platform_name + '-3-candidate.zip' From 3ebc508bde9ae4fb8b035239d0e378aac7e5b1f5 Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Wed, 10 Jul 2024 19:53:05 +0000 Subject: [PATCH 16/29] Fixing linting --- butler.py | 6 ++---- .../_internal/bot/tasks/update_task.py | 6 +++--- src/local/butler/deploy.py | 12 ++++++------ src/local/butler/package.py | 17 ++++++++--------- 4 files changed, 19 insertions(+), 22 deletions(-) diff --git a/butler.py b/butler.py index 169601a6d94..745ad36d87f 100644 --- a/butler.py +++ b/butler.py @@ -259,8 +259,7 @@ def main(): 'package', help='Package clusterfuzz with a staging revision') parser_package.add_argument( '-p', '--platform', choices=['linux', 'macos', 'windows', 'all']) - parser_package.add_argument( - '-r', '--release', choices=['prod', 'candidate']) + parser_package.add_argument('-r', '--release', choices=['prod', 'candidate']) parser_deploy = subparsers.add_parser('deploy', help='Deploy to Appengine') parser_deploy.add_argument( @@ -276,8 +275,7 @@ def main(): '--prod', action='store_true', help='Deploy to production.') parser_deploy.add_argument( '--targets', nargs='*', default=['appengine', 'k8s', 'zips']) - parser_deploy.add_argument( - '--release', '-r', choices=['prod', 'candidate']) + parser_deploy.add_argument('--release', '-r', choices=['prod', 'candidate']) parser_run_server = subparsers.add_parser( 'run_server', help='Run the local Clusterfuzz server.') diff --git a/src/clusterfuzz/_internal/bot/tasks/update_task.py b/src/clusterfuzz/_internal/bot/tasks/update_task.py index 0992c35a8fa..a487ebae19a 100644 --- a/src/clusterfuzz/_internal/bot/tasks/update_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/update_task.py @@ -39,20 +39,20 @@ from clusterfuzz._internal.system import process_handler from clusterfuzz._internal.system import shell -CLUSTERFUZZ_RELEASE = os.getenv['CLUSTERFUZZ_RELEASE'] -if CLUSTERFUZZ_RELEASE == None: +CLUSTERFUZZ_RELEASE = os.getenv('CLUSTERFUZZ_RELEASE') +if not CLUSTERFUZZ_RELEASE: CLUSTERFUZZ_RELEASE = 'prod' TESTS_LAST_UPDATE_KEY = 'tests_last_update' TESTS_UPDATE_INTERVAL_DAYS = 1 - REMOTE_MANIFEST_FILENAME = 'clusterfuzz-source.manifest' if sys.version_info.major == 3: REMOTE_MANIFEST_FILENAME += '.3' if CLUSTERFUZZ_RELEASE == 'candidate': REMOTE_MANIFEST_FILENAME += '-candidate' + def _rename_dll_for_update(absolute_filepath): """Rename a DLL to allow for updates.""" backup_filepath = absolute_filepath + '.bak.' + str(int(time.time())) diff --git a/src/local/butler/deploy.py b/src/local/butler/deploy.py index 6af96323cd2..02a5aa227fa 100644 --- a/src/local/butler/deploy.py +++ b/src/local/butler/deploy.py @@ -229,7 +229,10 @@ def _deploy_zip(bucket_name, zip_path, test_deployment=False): os.path.basename(zip_path))) -def _deploy_manifest(bucket_name, manifest_path, test_deployment=False, release='prod'): +def _deploy_manifest(bucket_name, + manifest_path, + test_deployment=False, + release='prod'): """Deploy source manifest to GCS.""" if sys.version_info.major == 3: if release == 'prod': @@ -402,7 +405,6 @@ def _prod_deployment_helper(config_dir, package_zip_paths, deploy_appengine=True, deploy_k8s=True, - python3=True, test_deployment=False, release='prod'): """Helper for production deployment.""" @@ -542,17 +544,16 @@ def execute(args): release = args.release if args.release == 'candidate': - release='candidate' + release = 'candidate' else: release = 'prod' - is_python3 = sys.version_info.major == 3 package_zip_paths = [] if deploy_zips: for platform_name in platforms: package_zip_paths.append( package.package( - revision, platform_name=platform_name, python3=is_python3, release=release)) + revision, platform_name=platform_name, release=release)) else: # package.package calls these, so only set these up if we're not packaging, # since they can be fairly slow. @@ -576,7 +577,6 @@ def execute(args): package_zip_paths, deploy_appengine, deploy_k8s, - python3=is_python3, test_deployment=test_deployment, release=release) diff --git a/src/local/butler/package.py b/src/local/butler/package.py index c4c963369b7..ed7b41e2fa8 100644 --- a/src/local/butler/package.py +++ b/src/local/butler/package.py @@ -71,8 +71,7 @@ def package(revision, target_zip_dir=constants.PACKAGE_TARGET_ZIP_DIRECTORY, target_manifest_path=constants.PACKAGE_TARGET_MANIFEST_PATH, platform_name=None, - release='prod', - python3=True): + release='prod'): """Prepare clusterfuzz-source.zip.""" is_ci = os.getenv('TEST_BOT_ENVIRONMENT') if not is_ci and common.is_git_dirty(): @@ -97,13 +96,10 @@ def package(revision, target_zip_name = constants.LEGACY_ZIP_NAME if platform_name: - if python3: - if release == 'prod': - target_zip_name = platform_name + '-3.zip' - else: - target_zip_name = platform_name + '-3-candidate.zip' + if release == 'prod': + target_zip_name = platform_name + '-3.zip' else: - target_zip_name = platform_name + '.zip' + target_zip_name = platform_name + '-3-candidate.zip' target_zip_path = os.path.join(target_zip_dir, target_zip_name) _clear_zip(target_zip_path) @@ -144,6 +140,7 @@ def package(revision, def execute(args): + """Execute the butler package command.""" release = 'prod' if args.release == 'candidate': release = 'candidate' @@ -155,4 +152,6 @@ def execute(args): release=release) else: package( - revision=common.compute_staging_revision(), platform_name=args.platform, release=release) + revision=common.compute_staging_revision(), + platform_name=args.platform, + release=release) From 379948951fca121054cb913c7d4c18130d11daf5 Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Wed, 10 Jul 2024 20:35:21 +0000 Subject: [PATCH 17/29] Fixing log emit tests due to logging new fields --- src/clusterfuzz/_internal/tests/core/metrics/logs_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/clusterfuzz/_internal/tests/core/metrics/logs_test.py b/src/clusterfuzz/_internal/tests/core/metrics/logs_test.py index 3fd688dbd76..e8aa8d70457 100644 --- a/src/clusterfuzz/_internal/tests/core/metrics/logs_test.py +++ b/src/clusterfuzz/_internal/tests/core/metrics/logs_test.py @@ -315,6 +315,8 @@ def test_info(self): 'target': 'bot', 'test': 'yes' }, + 'release': 'prod', + 'docker_image': '', 'location': { 'path': os.path.abspath(__file__).rstrip('c'), 'line': statement_line, @@ -339,6 +341,8 @@ def test_error(self): 'target': 'bot', 'test': 'yes' }, + 'release': 'prod', + 'docker_image': '', 'location': { 'path': os.path.abspath(__file__).rstrip('c'), 'line': statement_line, From 8d38f5b955dccff45d4a784cf2948e71188f8354 Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Thu, 11 Jul 2024 13:20:43 +0000 Subject: [PATCH 18/29] Adding the conditional deployment zip choice to all docker images --- docker/base/Dockerfile | 2 +- docker/chromium/base/start.sh | 6 ++++++ docker/chromium/builder/start.sh | 6 ++++++ docker/chromium/tests-syncer/start.sh | 6 ++++++ docker/fuchsia/start.sh | 6 ++++++ docker/oss-fuzz/base/start.sh | 6 ++++++ docker/oss-fuzz/host/Dockerfile | 6 ++++++ 7 files changed, 37 insertions(+), 1 deletion(-) diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index b9bac2e120b..2561418161f 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -151,5 +151,5 @@ RUN cd /data && \ # Make pip3.7 the default so that pipenv install --system works. mv /usr/local/bin/pip3.7 /usr/local/bin/pip && \ pipenv install --deploy --system - + CMD ["bash", "-ex", "/data/start.sh"] diff --git a/docker/chromium/base/start.sh b/docker/chromium/base/start.sh index 7c09640cccb..f6a2198ca79 100644 --- a/docker/chromium/base/start.sh +++ b/docker/chromium/base/start.sh @@ -16,4 +16,10 @@ source /data/setup.sh source /data/setup_clusterfuzz.sh +DEPLOYMENT_ZIP="linux3.zip" +if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then + DEPLOYMENT_ZIP="linux3-candidate.zip" +fi +export DEPLOYMENT_ZIP + bash -ex /data/start_clusterfuzz.sh diff --git a/docker/chromium/builder/start.sh b/docker/chromium/builder/start.sh index 9fc2e010f2e..ff7793efa5a 100644 --- a/docker/chromium/builder/start.sh +++ b/docker/chromium/builder/start.sh @@ -18,4 +18,10 @@ source /data/setup_depot_tools.sh source /data/setup_gerrit.sh source /data/setup_clusterfuzz.sh +DEPLOYMENT_ZIP="linux3.zip" +if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then + DEPLOYMENT_ZIP="linux3-candidate.zip" +fi +export DEPLOYMENT_ZIP + bash -ex /data/start_clusterfuzz.sh diff --git a/docker/chromium/tests-syncer/start.sh b/docker/chromium/tests-syncer/start.sh index 9fc2e010f2e..ff7793efa5a 100644 --- a/docker/chromium/tests-syncer/start.sh +++ b/docker/chromium/tests-syncer/start.sh @@ -18,4 +18,10 @@ source /data/setup_depot_tools.sh source /data/setup_gerrit.sh source /data/setup_clusterfuzz.sh +DEPLOYMENT_ZIP="linux3.zip" +if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then + DEPLOYMENT_ZIP="linux3-candidate.zip" +fi +export DEPLOYMENT_ZIP + bash -ex /data/start_clusterfuzz.sh diff --git a/docker/fuchsia/start.sh b/docker/fuchsia/start.sh index 300f6b65eb7..bc061d5392f 100644 --- a/docker/fuchsia/start.sh +++ b/docker/fuchsia/start.sh @@ -17,4 +17,10 @@ chmod 666 /dev/kvm source /data/setup_common.sh source /data/setup_clusterfuzz.sh +DEPLOYMENT_ZIP="linux3.zip" +if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then + DEPLOYMENT_ZIP="linux3-candidate.zip" +fi +export DEPLOYMENT_ZIP + bash -ex /data/start_clusterfuzz.sh diff --git a/docker/oss-fuzz/base/start.sh b/docker/oss-fuzz/base/start.sh index 64029102035..9162c612f6e 100644 --- a/docker/oss-fuzz/base/start.sh +++ b/docker/oss-fuzz/base/start.sh @@ -16,4 +16,10 @@ source /data/setup_common.sh source /data/setup_clusterfuzz.sh +DEPLOYMENT_ZIP="linux3.zip" +if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then + DEPLOYMENT_ZIP="linux3-candidate.zip" +fi +export DEPLOYMENT_ZIP + bash -ex /data/start_clusterfuzz.sh diff --git a/docker/oss-fuzz/host/Dockerfile b/docker/oss-fuzz/host/Dockerfile index 3d0eaa47a66..9d3db54987e 100644 --- a/docker/oss-fuzz/host/Dockerfile +++ b/docker/oss-fuzz/host/Dockerfile @@ -20,4 +20,10 @@ ENV NUM_WORKERS_PER_HOST 16 ENV DISABLE_MOUNTS True ENV TRUSTED_HOST True +DEPLOYMENT_ZIP="linux3.zip" +if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then + DEPLOYMENT_ZIP="linux3-candidate.zip" +fi +export DEPLOYMENT_ZIP + ENV RUN_CMD "python3.7 /data/start_host.py" From 6bd17b0112644939c9e8b5a42f8abe06f46a55f7 Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Thu, 11 Jul 2024 13:43:58 +0000 Subject: [PATCH 19/29] Temporarily changing the way docker images are built --- docker/build.sh | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/docker/build.sh b/docker/build.sh index a72a39f13a1..7cfd8a2ae6a 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -15,28 +15,29 @@ IMAGES=( gcr.io/clusterfuzz-images/base - gcr.io/clusterfuzz-images/high-end + #gcr.io/clusterfuzz-images/high-end gcr.io/clusterfuzz-images/chromium/base - gcr.io/clusterfuzz-images/chromium/builder - gcr.io/clusterfuzz-images/chromium/high-end + #gcr.io/clusterfuzz-images/chromium/builder + #gcr.io/clusterfuzz-images/chromium/high-end gcr.io/clusterfuzz-images/chromium/tester - gcr.io/clusterfuzz-images/chromium/tests-syncer - gcr.io/clusterfuzz-images/oss-fuzz/base - gcr.io/clusterfuzz-images/oss-fuzz/host - gcr.io/clusterfuzz-images/oss-fuzz/host-high-end - gcr.io/clusterfuzz-images/oss-fuzz/worker - gcr.io/clusterfuzz-images/ci - gcr.io/clusterfuzz-images/utask-main-scheduler - gcr.io/clusterfuzz-images/fuchsia + #gcr.io/clusterfuzz-images/chromium/tests-syncer + #gcr.io/clusterfuzz-images/oss-fuzz/base + #gcr.io/clusterfuzz-images/oss-fuzz/host + #gcr.io/clusterfuzz-images/oss-fuzz/host-high-end + #gcr.io/clusterfuzz-images/oss-fuzz/worker + #gcr.io/clusterfuzz-images/ci + #gcr.io/clusterfuzz-images/utask-main-scheduler + #gcr.io/clusterfuzz-images/fuchsia ) function docker_push { - docker push $image + #docker push $image docker push $image:$stamp } GIT_HASH=`git rev-parse HEAD | head -c7` -stamp=$GIT_HASH-$(date -u +%Y%m%d%H%M) +#stamp=$GIT_HASH-$(date -u +%Y%m%d%H%M) +stamp=vguidi_progrollout_v0 for image in "${IMAGES[@]}"; do docker build -t $image ${image#gcr.io/clusterfuzz-images/} docker tag $image $image:$stamp From 9f34c68aa3278bd6de6dd23fc4a3acdf1ed91791 Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Thu, 11 Jul 2024 17:17:31 +0000 Subject: [PATCH 20/29] Exporting deployment_zip before it is consumed in start.sh --- docker/chromium/base/start.sh | 6 ++++-- docker/chromium/builder/start.sh | 10 +++++----- docker/chromium/tests-syncer/start.sh | 9 +++++---- docker/fuchsia/start.sh | 9 +++++---- docker/oss-fuzz/base/start.sh | 6 +++--- 5 files changed, 22 insertions(+), 18 deletions(-) diff --git a/docker/chromium/base/start.sh b/docker/chromium/base/start.sh index f6a2198ca79..5008435f940 100644 --- a/docker/chromium/base/start.sh +++ b/docker/chromium/base/start.sh @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -source /data/setup.sh -source /data/setup_clusterfuzz.sh DEPLOYMENT_ZIP="linux3.zip" if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then @@ -22,4 +20,8 @@ if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then fi export DEPLOYMENT_ZIP +source /data/setup.sh +source /data/setup_clusterfuzz.sh + + bash -ex /data/start_clusterfuzz.sh diff --git a/docker/chromium/builder/start.sh b/docker/chromium/builder/start.sh index ff7793efa5a..3f18964e7e7 100644 --- a/docker/chromium/builder/start.sh +++ b/docker/chromium/builder/start.sh @@ -13,15 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -source /data/setup_common.sh -source /data/setup_depot_tools.sh -source /data/setup_gerrit.sh -source /data/setup_clusterfuzz.sh - DEPLOYMENT_ZIP="linux3.zip" if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then DEPLOYMENT_ZIP="linux3-candidate.zip" fi export DEPLOYMENT_ZIP +source /data/setup_common.sh +source /data/setup_depot_tools.sh +source /data/setup_gerrit.sh +source /data/setup_clusterfuzz.sh + bash -ex /data/start_clusterfuzz.sh diff --git a/docker/chromium/tests-syncer/start.sh b/docker/chromium/tests-syncer/start.sh index ff7793efa5a..a64839e83ba 100644 --- a/docker/chromium/tests-syncer/start.sh +++ b/docker/chromium/tests-syncer/start.sh @@ -13,10 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -source /data/setup_common.sh -source /data/setup_depot_tools.sh -source /data/setup_gerrit.sh -source /data/setup_clusterfuzz.sh DEPLOYMENT_ZIP="linux3.zip" if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then @@ -24,4 +20,9 @@ if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then fi export DEPLOYMENT_ZIP +source /data/setup_common.sh +source /data/setup_depot_tools.sh +source /data/setup_gerrit.sh +source /data/setup_clusterfuzz.sh + bash -ex /data/start_clusterfuzz.sh diff --git a/docker/fuchsia/start.sh b/docker/fuchsia/start.sh index bc061d5392f..c6a3b07be72 100644 --- a/docker/fuchsia/start.sh +++ b/docker/fuchsia/start.sh @@ -13,14 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -chmod 666 /dev/kvm -source /data/setup_common.sh -source /data/setup_clusterfuzz.sh - DEPLOYMENT_ZIP="linux3.zip" if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then DEPLOYMENT_ZIP="linux3-candidate.zip" fi export DEPLOYMENT_ZIP + +chmod 666 /dev/kvm +source /data/setup_common.sh +source /data/setup_clusterfuzz.sh + bash -ex /data/start_clusterfuzz.sh diff --git a/docker/oss-fuzz/base/start.sh b/docker/oss-fuzz/base/start.sh index 9162c612f6e..e0943f6f68d 100644 --- a/docker/oss-fuzz/base/start.sh +++ b/docker/oss-fuzz/base/start.sh @@ -13,13 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -source /data/setup_common.sh -source /data/setup_clusterfuzz.sh - DEPLOYMENT_ZIP="linux3.zip" if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then DEPLOYMENT_ZIP="linux3-candidate.zip" fi export DEPLOYMENT_ZIP +source /data/setup_common.sh +source /data/setup_clusterfuzz.sh + bash -ex /data/start_clusterfuzz.sh From f5863e8a40876737d2a4b9d3f4088c189765fff7 Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Thu, 11 Jul 2024 17:26:08 +0000 Subject: [PATCH 21/29] Renaming deployment file from linux3.zip to linux-3.zip --- docker/base/start.sh | 4 ++-- docker/chromium/base/start.sh | 4 ++-- docker/chromium/builder/start.sh | 4 ++-- docker/chromium/tests-syncer/start.sh | 2 +- docker/fuchsia/start.sh | 4 ++-- docker/oss-fuzz/base/start.sh | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docker/base/start.sh b/docker/base/start.sh index e0943f6f68d..5dfac075f2a 100644 --- a/docker/base/start.sh +++ b/docker/base/start.sh @@ -13,9 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -DEPLOYMENT_ZIP="linux3.zip" +DEPLOYMENT_ZIP="linux-3.zip" if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then - DEPLOYMENT_ZIP="linux3-candidate.zip" + DEPLOYMENT_ZIP="linux-3-candidate.zip" fi export DEPLOYMENT_ZIP diff --git a/docker/chromium/base/start.sh b/docker/chromium/base/start.sh index 5008435f940..d1111a95cc7 100644 --- a/docker/chromium/base/start.sh +++ b/docker/chromium/base/start.sh @@ -14,9 +14,9 @@ # limitations under the License. -DEPLOYMENT_ZIP="linux3.zip" +DEPLOYMENT_ZIP="linux-3.zip" if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then - DEPLOYMENT_ZIP="linux3-candidate.zip" + DEPLOYMENT_ZIP="linux-3-candidate.zip" fi export DEPLOYMENT_ZIP diff --git a/docker/chromium/builder/start.sh b/docker/chromium/builder/start.sh index 3f18964e7e7..d80f401410f 100644 --- a/docker/chromium/builder/start.sh +++ b/docker/chromium/builder/start.sh @@ -13,9 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -DEPLOYMENT_ZIP="linux3.zip" +DEPLOYMENT_ZIP="linux-3.zip" if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then - DEPLOYMENT_ZIP="linux3-candidate.zip" + DEPLOYMENT_ZIP="linux-3-candidate.zip" fi export DEPLOYMENT_ZIP diff --git a/docker/chromium/tests-syncer/start.sh b/docker/chromium/tests-syncer/start.sh index a64839e83ba..85ae6eaffd7 100644 --- a/docker/chromium/tests-syncer/start.sh +++ b/docker/chromium/tests-syncer/start.sh @@ -16,7 +16,7 @@ DEPLOYMENT_ZIP="linux3.zip" if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then - DEPLOYMENT_ZIP="linux3-candidate.zip" + DEPLOYMENT_ZIP="linux-3-candidate.zip" fi export DEPLOYMENT_ZIP diff --git a/docker/fuchsia/start.sh b/docker/fuchsia/start.sh index c6a3b07be72..bd6ed97aecb 100644 --- a/docker/fuchsia/start.sh +++ b/docker/fuchsia/start.sh @@ -13,9 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -DEPLOYMENT_ZIP="linux3.zip" +DEPLOYMENT_ZIP="linux-3.zip" if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then - DEPLOYMENT_ZIP="linux3-candidate.zip" + DEPLOYMENT_ZIP="linux-3-candidate.zip" fi export DEPLOYMENT_ZIP diff --git a/docker/oss-fuzz/base/start.sh b/docker/oss-fuzz/base/start.sh index e0943f6f68d..5dfac075f2a 100644 --- a/docker/oss-fuzz/base/start.sh +++ b/docker/oss-fuzz/base/start.sh @@ -13,9 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -DEPLOYMENT_ZIP="linux3.zip" +DEPLOYMENT_ZIP="linux-3.zip" if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then - DEPLOYMENT_ZIP="linux3-candidate.zip" + DEPLOYMENT_ZIP="linux-3-candidate.zip" fi export DEPLOYMENT_ZIP From cda380bf6c7abf4367cad0891fd9dd0e4bbcd729 Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Thu, 11 Jul 2024 17:28:12 +0000 Subject: [PATCH 22/29] Removing other references to linux3.zip --- docker/chromium/tests-syncer/start.sh | 2 +- docker/oss-fuzz/host/Dockerfile | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/docker/chromium/tests-syncer/start.sh b/docker/chromium/tests-syncer/start.sh index 85ae6eaffd7..f38a894e0de 100644 --- a/docker/chromium/tests-syncer/start.sh +++ b/docker/chromium/tests-syncer/start.sh @@ -14,7 +14,7 @@ # limitations under the License. -DEPLOYMENT_ZIP="linux3.zip" +DEPLOYMENT_ZIP="linux-3.zip" if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then DEPLOYMENT_ZIP="linux-3-candidate.zip" fi diff --git a/docker/oss-fuzz/host/Dockerfile b/docker/oss-fuzz/host/Dockerfile index 9d3db54987e..61488c6ed06 100644 --- a/docker/oss-fuzz/host/Dockerfile +++ b/docker/oss-fuzz/host/Dockerfile @@ -20,10 +20,5 @@ ENV NUM_WORKERS_PER_HOST 16 ENV DISABLE_MOUNTS True ENV TRUSTED_HOST True -DEPLOYMENT_ZIP="linux3.zip" -if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then - DEPLOYMENT_ZIP="linux3-candidate.zip" -fi -export DEPLOYMENT_ZIP ENV RUN_CMD "python3.7 /data/start_host.py" From 5d28a04df0ca67f3495172d9fae0ef4cc2c8f654 Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Thu, 11 Jul 2024 20:41:15 +0000 Subject: [PATCH 23/29] Fixing env var names and bash syntax --- docker/base/setup_clusterfuzz.sh | 8 ++++++-- docker/base/start.sh | 2 +- docker/chromium/base/start.sh | 7 ++++++- docker/chromium/builder/start.sh | 2 +- docker/chromium/tests-syncer/start.sh | 3 +-- docker/fuchsia/start.sh | 2 +- docker/oss-fuzz/base/start.sh | 2 +- src/clusterfuzz/_internal/metrics/logs.py | 4 ++-- 8 files changed, 19 insertions(+), 11 deletions(-) diff --git a/docker/base/setup_clusterfuzz.sh b/docker/base/setup_clusterfuzz.sh index 0f18345a53c..af4ea3c539d 100644 --- a/docker/base/setup_clusterfuzz.sh +++ b/docker/base/setup_clusterfuzz.sh @@ -18,6 +18,8 @@ if [ -z "$DEPLOYMENT_BUCKET" ]; then export DEPLOYMENT_BUCKET=$(curl -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/project/attributes/deployment-bucket) fi +CLUSTERFUZZ_FILE=clusterfuzz_package.zip + # When $LOCAL_SRC is set, use source zip on mounted volume for local testing. if [[ -z "$LOCAL_SRC" ]]; then # Set up ClusterFuzz @@ -25,8 +27,10 @@ if [[ -z "$LOCAL_SRC" ]]; then rm -rf clusterfuzz fi - gsutil cp gs://$DEPLOYMENT_BUCKET/$DEPLOYMENT_ZIP . - unzip -q -o $DEPLOYMENT_ZIP + # DEPLOYMENT_ZIP might be test-deployment/linux-3.zip, so we do not extract DEPLOYMENT_ZIP directly + + gsutil cp gs://$DEPLOYMENT_BUCKET/$DEPLOYMENT_ZIP $CLUSTERFUZZ_FILE + unzip -q -o $CLUSTERFUZZ_FILE fi # Some configurations (e.g. hosts) run many instances of ClusterFuzz. Don't diff --git a/docker/base/start.sh b/docker/base/start.sh index 5dfac075f2a..1ff46017433 100644 --- a/docker/base/start.sh +++ b/docker/base/start.sh @@ -14,7 +14,7 @@ # limitations under the License. DEPLOYMENT_ZIP="linux-3.zip" -if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then +if [[ $CLUSTERFUZZ_RELEASE == "candidate" ]]; then DEPLOYMENT_ZIP="linux-3-candidate.zip" fi export DEPLOYMENT_ZIP diff --git a/docker/chromium/base/start.sh b/docker/chromium/base/start.sh index d1111a95cc7..6d0afdbd1b1 100644 --- a/docker/chromium/base/start.sh +++ b/docker/chromium/base/start.sh @@ -15,9 +15,14 @@ DEPLOYMENT_ZIP="linux-3.zip" -if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then +if [[ $CLUSTERFUZZ_RELEASE == "candidate" ]]; then DEPLOYMENT_ZIP="linux-3-candidate.zip" fi + +if [ -n $USE_TEST_DEPLOYMENT ]; then + DEPLOYMENT_ZIP="test-deployment/$DEPLOYMENT_ZIP" +fi + export DEPLOYMENT_ZIP source /data/setup.sh diff --git a/docker/chromium/builder/start.sh b/docker/chromium/builder/start.sh index d80f401410f..c17641c3e9a 100644 --- a/docker/chromium/builder/start.sh +++ b/docker/chromium/builder/start.sh @@ -14,7 +14,7 @@ # limitations under the License. DEPLOYMENT_ZIP="linux-3.zip" -if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then +if [[ $CLUSTERFUZZ_RELEASE == "candidate" ]]; then DEPLOYMENT_ZIP="linux-3-candidate.zip" fi export DEPLOYMENT_ZIP diff --git a/docker/chromium/tests-syncer/start.sh b/docker/chromium/tests-syncer/start.sh index f38a894e0de..c17641c3e9a 100644 --- a/docker/chromium/tests-syncer/start.sh +++ b/docker/chromium/tests-syncer/start.sh @@ -13,9 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. - DEPLOYMENT_ZIP="linux-3.zip" -if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then +if [[ $CLUSTERFUZZ_RELEASE == "candidate" ]]; then DEPLOYMENT_ZIP="linux-3-candidate.zip" fi export DEPLOYMENT_ZIP diff --git a/docker/fuchsia/start.sh b/docker/fuchsia/start.sh index bd6ed97aecb..f46836f782f 100644 --- a/docker/fuchsia/start.sh +++ b/docker/fuchsia/start.sh @@ -14,7 +14,7 @@ # limitations under the License. DEPLOYMENT_ZIP="linux-3.zip" -if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then +if [[ $CLUSTERFUZZ_RELEASE == "candidate" ]]; then DEPLOYMENT_ZIP="linux-3-candidate.zip" fi export DEPLOYMENT_ZIP diff --git a/docker/oss-fuzz/base/start.sh b/docker/oss-fuzz/base/start.sh index 5dfac075f2a..1ff46017433 100644 --- a/docker/oss-fuzz/base/start.sh +++ b/docker/oss-fuzz/base/start.sh @@ -14,7 +14,7 @@ # limitations under the License. DEPLOYMENT_ZIP="linux-3.zip" -if [[$CLUSTERFUZZ_RELEASE == "candidate" ]]; then +if [[ $CLUSTERFUZZ_RELEASE == "candidate" ]]; then DEPLOYMENT_ZIP="linux-3-candidate.zip" fi export DEPLOYMENT_ZIP diff --git a/src/clusterfuzz/_internal/metrics/logs.py b/src/clusterfuzz/_internal/metrics/logs.py index bf4ad9ee7d4..6a69e8fb5cb 100644 --- a/src/clusterfuzz/_internal/metrics/logs.py +++ b/src/clusterfuzz/_internal/metrics/logs.py @@ -554,8 +554,8 @@ def emit(level, message, exc_info=None, **extras): 'line': line_number, 'method': method_name }, - 'release': os.environ.get('CLUSTERFUZZ_STAGE', 'prod'), - 'docker_image': os.environ.get('IMAGE', '') + 'release': os.environ.get('CLUSTERFUZZ_RELEASE', 'prod'), + 'docker_image': os.environ.get('DOCKER_IMAGE', '') }) From 871fba407dd6e76b329bf8ee65a60fb5909d2fd6 Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Thu, 11 Jul 2024 20:43:51 +0000 Subject: [PATCH 24/29] Fixing operator to check unset env var --- docker/chromium/base/start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/chromium/base/start.sh b/docker/chromium/base/start.sh index 6d0afdbd1b1..cd49ca88618 100644 --- a/docker/chromium/base/start.sh +++ b/docker/chromium/base/start.sh @@ -19,7 +19,7 @@ if [[ $CLUSTERFUZZ_RELEASE == "candidate" ]]; then DEPLOYMENT_ZIP="linux-3-candidate.zip" fi -if [ -n $USE_TEST_DEPLOYMENT ]; then +if [ ! -z $USE_TEST_DEPLOYMENT ]; then DEPLOYMENT_ZIP="test-deployment/$DEPLOYMENT_ZIP" fi From 4556d5cd2c956ff5912bc1d5d32ea41974c94de6 Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Mon, 22 Jul 2024 19:52:04 +0000 Subject: [PATCH 25/29] Fixing nits from Oliver's review --- butler.py | 4 +-- docker/base/setup_clusterfuzz.sh | 2 -- docker/build.sh | 25 +++++++++---------- .../_internal/bot/tasks/update_task.py | 4 +-- src/local/butler/deploy.py | 8 +----- src/local/butler/package.py | 7 ++---- 6 files changed, 18 insertions(+), 32 deletions(-) diff --git a/butler.py b/butler.py index 745ad36d87f..5ff042db472 100644 --- a/butler.py +++ b/butler.py @@ -259,7 +259,7 @@ def main(): 'package', help='Package clusterfuzz with a staging revision') parser_package.add_argument( '-p', '--platform', choices=['linux', 'macos', 'windows', 'all']) - parser_package.add_argument('-r', '--release', choices=['prod', 'candidate']) + parser_package.add_argument('-r', '--release', choices=['prod', 'candidate'], default='prod') parser_deploy = subparsers.add_parser('deploy', help='Deploy to Appengine') parser_deploy.add_argument( @@ -275,7 +275,7 @@ def main(): '--prod', action='store_true', help='Deploy to production.') parser_deploy.add_argument( '--targets', nargs='*', default=['appengine', 'k8s', 'zips']) - parser_deploy.add_argument('--release', '-r', choices=['prod', 'candidate']) + parser_deploy.add_argument('--release', '-r', choices=['prod', 'candidate'], default='prod') parser_run_server = subparsers.add_parser( 'run_server', help='Run the local Clusterfuzz server.') diff --git a/docker/base/setup_clusterfuzz.sh b/docker/base/setup_clusterfuzz.sh index af4ea3c539d..8b92e7e52fc 100644 --- a/docker/base/setup_clusterfuzz.sh +++ b/docker/base/setup_clusterfuzz.sh @@ -19,7 +19,6 @@ if [ -z "$DEPLOYMENT_BUCKET" ]; then fi CLUSTERFUZZ_FILE=clusterfuzz_package.zip - # When $LOCAL_SRC is set, use source zip on mounted volume for local testing. if [[ -z "$LOCAL_SRC" ]]; then # Set up ClusterFuzz @@ -28,7 +27,6 @@ if [[ -z "$LOCAL_SRC" ]]; then fi # DEPLOYMENT_ZIP might be test-deployment/linux-3.zip, so we do not extract DEPLOYMENT_ZIP directly - gsutil cp gs://$DEPLOYMENT_BUCKET/$DEPLOYMENT_ZIP $CLUSTERFUZZ_FILE unzip -q -o $CLUSTERFUZZ_FILE fi diff --git a/docker/build.sh b/docker/build.sh index 7cfd8a2ae6a..085be6655ee 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -15,19 +15,19 @@ IMAGES=( gcr.io/clusterfuzz-images/base - #gcr.io/clusterfuzz-images/high-end + gcr.io/clusterfuzz-images/high-end gcr.io/clusterfuzz-images/chromium/base - #gcr.io/clusterfuzz-images/chromium/builder - #gcr.io/clusterfuzz-images/chromium/high-end + gcr.io/clusterfuzz-images/chromium/builder + gcr.io/clusterfuzz-images/chromium/high-end gcr.io/clusterfuzz-images/chromium/tester - #gcr.io/clusterfuzz-images/chromium/tests-syncer - #gcr.io/clusterfuzz-images/oss-fuzz/base - #gcr.io/clusterfuzz-images/oss-fuzz/host - #gcr.io/clusterfuzz-images/oss-fuzz/host-high-end - #gcr.io/clusterfuzz-images/oss-fuzz/worker - #gcr.io/clusterfuzz-images/ci - #gcr.io/clusterfuzz-images/utask-main-scheduler - #gcr.io/clusterfuzz-images/fuchsia + gcr.io/clusterfuzz-images/chromium/tests-syncer + gcr.io/clusterfuzz-images/oss-fuzz/base + gcr.io/clusterfuzz-images/oss-fuzz/host + gcr.io/clusterfuzz-images/oss-fuzz/host-high-end + gcr.io/clusterfuzz-images/oss-fuzz/worker + gcr.io/clusterfuzz-images/ci + gcr.io/clusterfuzz-images/utask-main-scheduler + gcr.io/clusterfuzz-images/fuchsia ) function docker_push { @@ -36,8 +36,7 @@ function docker_push { } GIT_HASH=`git rev-parse HEAD | head -c7` -#stamp=$GIT_HASH-$(date -u +%Y%m%d%H%M) -stamp=vguidi_progrollout_v0 +stamp=$GIT_HASH-$(date -u +%Y%m%d%H%M) for image in "${IMAGES[@]}"; do docker build -t $image ${image#gcr.io/clusterfuzz-images/} docker tag $image $image:$stamp diff --git a/src/clusterfuzz/_internal/bot/tasks/update_task.py b/src/clusterfuzz/_internal/bot/tasks/update_task.py index a487ebae19a..8bf56b14a8a 100644 --- a/src/clusterfuzz/_internal/bot/tasks/update_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/update_task.py @@ -39,9 +39,7 @@ from clusterfuzz._internal.system import process_handler from clusterfuzz._internal.system import shell -CLUSTERFUZZ_RELEASE = os.getenv('CLUSTERFUZZ_RELEASE') -if not CLUSTERFUZZ_RELEASE: - CLUSTERFUZZ_RELEASE = 'prod' +CLUSTERFUZZ_RELEASE = os.getenv('CLUSTERFUZZ_RELEASE', 'prod') TESTS_LAST_UPDATE_KEY = 'tests_last_update' TESTS_UPDATE_INTERVAL_DAYS = 1 diff --git a/src/local/butler/deploy.py b/src/local/butler/deploy.py index 02a5aa227fa..e6bac6dc2e4 100644 --- a/src/local/butler/deploy.py +++ b/src/local/butler/deploy.py @@ -542,18 +542,12 @@ def execute(args): deploy_k8s = False deploy_zips = True - release = args.release - if args.release == 'candidate': - release = 'candidate' - else: - release = 'prod' - package_zip_paths = [] if deploy_zips: for platform_name in platforms: package_zip_paths.append( package.package( - revision, platform_name=platform_name, release=release)) + revision, platform_name=platform_name, release=args.release)) else: # package.package calls these, so only set these up if we're not packaging, # since they can be fairly slow. diff --git a/src/local/butler/package.py b/src/local/butler/package.py index ed7b41e2fa8..cd945c8a770 100644 --- a/src/local/butler/package.py +++ b/src/local/butler/package.py @@ -141,17 +141,14 @@ def package(revision, def execute(args): """Execute the butler package command.""" - release = 'prod' - if args.release == 'candidate': - release = 'candidate' if args.platform == 'all': for platform_name in list(constants.PLATFORMS.keys()): package( revision=common.compute_staging_revision(), platform_name=platform_name, - release=release) + release=args.release) else: package( revision=common.compute_staging_revision(), platform_name=args.platform, - release=release) + release=args.release) From dc071a613353aa805e86541406cf7496524a9f27 Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Mon, 22 Jul 2024 21:08:02 +0000 Subject: [PATCH 26/29] Moving manifest/zip package filename resolution to util to share between update task and butler --- src/clusterfuzz/_internal/base/utils.py | 32 ++++++++++++++++ .../_internal/bot/tasks/update_task.py | 38 ++++++------------- src/local/butler/deploy.py | 13 ++----- src/local/butler/package.py | 6 +-- 4 files changed, 48 insertions(+), 41 deletions(-) diff --git a/src/clusterfuzz/_internal/base/utils.py b/src/clusterfuzz/_internal/base/utils.py index 6c00f79bd6a..afd1e3988e8 100644 --- a/src/clusterfuzz/_internal/base/utils.py +++ b/src/clusterfuzz/_internal/base/utils.py @@ -227,6 +227,38 @@ def get_application_id(): return app_id +def get_clusterfuzz_release(): + return os.getenv('CLUSTERFUZZ_RELEASE', 'prod') + +def _get_manifest_release_suffix(release): + suffix = '' + if sys.version_info.major == 3: + suffix += '.3' + if release == 'candidate': + suffix += '-candidate' + return suffix + + +def _get_deployment_zip_release_suffix(release): + suffix = '' + if sys.version_info.major == 3: + suffix += '-3' + if release == 'candidate': + suffix += '-candidate' + return suffix + + +def get_platform_deployment_filename(platform, release): + """Return the platform deployment filename.""" + # Expects linux, macos or windows + base_filename = platform + release_filename_suffix = _get_deployment_zip_release_suffix(release) + return f'{base_filename}{release_filename_suffix}.zip' + + +def get_remote_manifest_filename(release): + return f'clusterfuzz-source.manifest{_get_manifest_release_suffix(release)}' + def service_account_email(): """Get the service account name.""" diff --git a/src/clusterfuzz/_internal/bot/tasks/update_task.py b/src/clusterfuzz/_internal/bot/tasks/update_task.py index 8bf56b14a8a..a815bb66578 100644 --- a/src/clusterfuzz/_internal/bot/tasks/update_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/update_task.py @@ -39,17 +39,9 @@ from clusterfuzz._internal.system import process_handler from clusterfuzz._internal.system import shell -CLUSTERFUZZ_RELEASE = os.getenv('CLUSTERFUZZ_RELEASE', 'prod') - TESTS_LAST_UPDATE_KEY = 'tests_last_update' TESTS_UPDATE_INTERVAL_DAYS = 1 -REMOTE_MANIFEST_FILENAME = 'clusterfuzz-source.manifest' -if sys.version_info.major == 3: - REMOTE_MANIFEST_FILENAME += '.3' -if CLUSTERFUZZ_RELEASE == 'candidate': - REMOTE_MANIFEST_FILENAME += '-candidate' - def _rename_dll_for_update(absolute_filepath): """Rename a DLL to allow for updates.""" @@ -57,23 +49,6 @@ def _rename_dll_for_update(absolute_filepath): os.rename(absolute_filepath, backup_filepath) -def _platform_deployment_filename(): - """Return the platform deployment filename.""" - platform_mappings = { - 'Linux': 'linux', - 'Windows': 'windows', - 'Darwin': 'macos' - } - - base_filename = platform_mappings[platform.system()] - if sys.version_info.major == 3: - base_filename += '-3' - if CLUSTERFUZZ_RELEASE == 'candidate': - base_filename += '-candidate' - - return base_filename + '.zip' - - def _deployment_file_url(filename): """Helper to return deployment file url.""" deployment_bucket = local_config.ProjectConfig().get('deployment.bucket') @@ -87,12 +62,21 @@ def _deployment_file_url(filename): def get_source_url(): """Return the source URL.""" - return _deployment_file_url(_platform_deployment_filename()) + release = utils.get_clusterfuzz_release() + platform_name = platform.system() + platform_mappings = { + 'Linux': 'linux', + 'Windows': 'windows', + 'Darwin': 'macos' + } + platform_name= platform_mappings[platform_name] + return _deployment_file_url(utils.get_platform_deployment_filename(platform_name, release)) def get_source_manifest_url(): """Return the source manifest URL.""" - return _deployment_file_url(REMOTE_MANIFEST_FILENAME) + release = utils.get_clusterfuzz_release() + return _deployment_file_url(utils.get_remote_manifest_filename(release)) def clear_old_files(directory, extracted_file_set): diff --git a/src/local/butler/deploy.py b/src/local/butler/deploy.py index e6bac6dc2e4..05102b5aceb 100644 --- a/src/local/butler/deploy.py +++ b/src/local/butler/deploy.py @@ -27,6 +27,7 @@ from local.butler import common from local.butler import constants from local.butler import package +from clusterfuzz._internal.base import utils from src.clusterfuzz._internal.config import local_config from src.clusterfuzz._internal.system import environment @@ -234,15 +235,7 @@ def _deploy_manifest(bucket_name, test_deployment=False, release='prod'): """Deploy source manifest to GCS.""" - if sys.version_info.major == 3: - if release == 'prod': - manifest_suffix = '3' - else: - manifest_suffix = '3-candidate' - else: - manifest_suffix = '' - - remote_manifest_path = f'clusterfuzz-source.manifest.{manifest_suffix}' + remote_manifest_path = utils.get_remote_manifest_filename(release) if test_deployment: common.execute(f'gsutil cp {manifest_path} ' @@ -572,7 +565,7 @@ def execute(args): deploy_appengine, deploy_k8s, test_deployment=test_deployment, - release=release) + release=args.release) with open(constants.PACKAGE_TARGET_MANIFEST_PATH) as f: print('Source updated to %s' % f.read()) diff --git a/src/local/butler/package.py b/src/local/butler/package.py index cd945c8a770..5a1cf79f194 100644 --- a/src/local/butler/package.py +++ b/src/local/butler/package.py @@ -18,6 +18,7 @@ import sys import zipfile +from clusterfuzz._internal.base import utils from local.butler import appengine from local.butler import common from local.butler import constants @@ -96,10 +97,7 @@ def package(revision, target_zip_name = constants.LEGACY_ZIP_NAME if platform_name: - if release == 'prod': - target_zip_name = platform_name + '-3.zip' - else: - target_zip_name = platform_name + '-3-candidate.zip' + target_zip_name = utils.get_platform_deployment_filename(platform_name, release) target_zip_path = os.path.join(target_zip_dir, target_zip_name) _clear_zip(target_zip_path) From 3985c5ab039ab53bca6fb93ea7a9d305923557e9 Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Tue, 23 Jul 2024 15:54:16 +0000 Subject: [PATCH 27/29] Resolving Jonathan's nits --- docker/build.sh | 2 +- docker/oss-fuzz/host/Dockerfile | 1 - src/clusterfuzz/_internal/base/utils.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/build.sh b/docker/build.sh index 085be6655ee..a72a39f13a1 100755 --- a/docker/build.sh +++ b/docker/build.sh @@ -31,7 +31,7 @@ IMAGES=( ) function docker_push { - #docker push $image + docker push $image docker push $image:$stamp } diff --git a/docker/oss-fuzz/host/Dockerfile b/docker/oss-fuzz/host/Dockerfile index 61488c6ed06..3d0eaa47a66 100644 --- a/docker/oss-fuzz/host/Dockerfile +++ b/docker/oss-fuzz/host/Dockerfile @@ -20,5 +20,4 @@ ENV NUM_WORKERS_PER_HOST 16 ENV DISABLE_MOUNTS True ENV TRUSTED_HOST True - ENV RUN_CMD "python3.7 /data/start_host.py" diff --git a/src/clusterfuzz/_internal/base/utils.py b/src/clusterfuzz/_internal/base/utils.py index afd1e3988e8..ecb4befac29 100644 --- a/src/clusterfuzz/_internal/base/utils.py +++ b/src/clusterfuzz/_internal/base/utils.py @@ -250,7 +250,7 @@ def _get_deployment_zip_release_suffix(release): def get_platform_deployment_filename(platform, release): """Return the platform deployment filename.""" - # Expects linux, macos or windows + # Expects linux, macos or windows. base_filename = platform release_filename_suffix = _get_deployment_zip_release_suffix(release) return f'{base_filename}{release_filename_suffix}.zip' From 006675e51ae161083e99cab6b3c13c7151f773e8 Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Tue, 23 Jul 2024 18:47:16 +0000 Subject: [PATCH 28/29] Adding release information on update task logs, moving to f string format --- src/clusterfuzz/_internal/bot/tasks/update_task.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/clusterfuzz/_internal/bot/tasks/update_task.py b/src/clusterfuzz/_internal/bot/tasks/update_task.py index a815bb66578..ce7118ef5b5 100644 --- a/src/clusterfuzz/_internal/bot/tasks/update_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/update_task.py @@ -151,14 +151,14 @@ def get_newer_source_revision(): logs.info('No manifest found. Forcing an update.') return source_version - logs.info('Local source code version: %s.' % local_source_version) - logs.info('Remote source code version: %s.' % source_version) + logs.info(f'Local source code version: {local_source_version}.') + logs.info(f'Remote source code version: {source_version}.') if local_source_version >= source_version: logs.info('Remote souce code <= local source code. No update.') # No source code update found. Source code is current, bail out. return None - logs.info(f'New source code: {source_version}') + logs.info(f'New source code: {source_version} (updated from {local_source_version}, on release {utils.get_clusterfuzz_release()})') return source_version @@ -271,7 +271,7 @@ def update_source_code(): source_version = utils.read_data_from_file( local_manifest_path, eval_data=False).decode('utf-8').strip() os.remove(temp_archive) - logs.info('Source code updated to %s.' % source_version) + logs.info(f'Source code updated to {source_version} (release = {utils.get_clusterfuzz_release()}).') def update_tests_if_needed(): From 4f1b5eb0c3c9c012abe708f745343bd7b6f2bd22 Mon Sep 17 00:00:00 2001 From: Vitor Guidi Date: Tue, 23 Jul 2024 20:41:53 +0000 Subject: [PATCH 29/29] Adding logs to match a manifest to the local release --- src/clusterfuzz/_internal/bot/tasks/update_task.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/clusterfuzz/_internal/bot/tasks/update_task.py b/src/clusterfuzz/_internal/bot/tasks/update_task.py index ce7118ef5b5..3a462e66cf0 100644 --- a/src/clusterfuzz/_internal/bot/tasks/update_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/update_task.py @@ -151,8 +151,8 @@ def get_newer_source_revision(): logs.info('No manifest found. Forcing an update.') return source_version - logs.info(f'Local source code version: {local_source_version}.') - logs.info(f'Remote source code version: {source_version}.') + logs.info(f'Local source code version: {local_source_version}, on release {utils.get_clusterfuzz_release()}.') + logs.info(f'Remote source code version: {source_version}, on release {utils.get_clusterfuzz_release()}.') if local_source_version >= source_version: logs.info('Remote souce code <= local source code. No update.') # No source code update found. Source code is current, bail out.