diff --git a/butler.py b/butler.py index e0d51de4207..5ff042db472 100644 --- a/butler.py +++ b/butler.py @@ -259,6 +259,7 @@ def main(): 'package', help='Package clusterfuzz with a staging revision') parser_package.add_argument( '-p', '--platform', choices=['linux', 'macos', 'windows', 'all']) + parser_package.add_argument('-r', '--release', choices=['prod', 'candidate'], default='prod') parser_deploy = subparsers.add_parser('deploy', help='Deploy to Appengine') parser_deploy.add_argument( @@ -274,6 +275,7 @@ def main(): '--prod', action='store_true', help='Deploy to production.') parser_deploy.add_argument( '--targets', nargs='*', default=['appengine', 'k8s', 'zips']) + parser_deploy.add_argument('--release', '-r', choices=['prod', 'candidate'], default='prod') parser_run_server = subparsers.add_parser( 'run_server', help='Run the local Clusterfuzz server.') diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index eef5b4a2e7f..2561418161f 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -97,7 +97,7 @@ RUN wget https://download.java.net/java/GA/jdk15.0.2/0d1cfde4252546c6931946de8db tar -xzv --strip-components=1 -f openjdk-15.0.2_linux-x64_bin.tar.gz --directory $JAVA_HOME && \ rm -rf openjdk*.tar.gz $JAVA_HOME/jmods $JAVA_HOME/lib/src.zip -# Install Python 3. +# Install Python 3.7 RUN curl -sS https://www.python.org/ftp/python/3.7.7/Python-3.7.7.tgz | tar -C /tmp -xzv && \ cd /tmp/Python-3.7.7 && \ ./configure --enable-optimizations --enable-loadable-sqlite-extensions && make altinstall && \ @@ -131,7 +131,6 @@ ENV ROOT_DIR $INSTALL_DIRECTORY/clusterfuzz ENV UPDATE_WEB_TESTS True ENV PYTHONPATH $INSTALL_DIRECTORY/clusterfuzz/src ENV RUN_CMD "python3.7 $ROOT_DIR/src/python/bot/startup/run.py" -ENV DEPLOYMENT_ZIP "linux-3.zip" # Passwordless sudo (needed for AFL launcher). RUN groupadd nopwsudo && \ diff --git a/docker/base/setup_clusterfuzz.sh b/docker/base/setup_clusterfuzz.sh index 0f18345a53c..8b92e7e52fc 100644 --- a/docker/base/setup_clusterfuzz.sh +++ b/docker/base/setup_clusterfuzz.sh @@ -18,6 +18,7 @@ if [ -z "$DEPLOYMENT_BUCKET" ]; then export DEPLOYMENT_BUCKET=$(curl -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/project/attributes/deployment-bucket) fi +CLUSTERFUZZ_FILE=clusterfuzz_package.zip # When $LOCAL_SRC is set, use source zip on mounted volume for local testing. if [[ -z "$LOCAL_SRC" ]]; then # Set up ClusterFuzz @@ -25,8 +26,9 @@ if [[ -z "$LOCAL_SRC" ]]; then rm -rf clusterfuzz fi - gsutil cp gs://$DEPLOYMENT_BUCKET/$DEPLOYMENT_ZIP . - unzip -q -o $DEPLOYMENT_ZIP + # DEPLOYMENT_ZIP might be test-deployment/linux-3.zip, so we do not extract DEPLOYMENT_ZIP directly + gsutil cp gs://$DEPLOYMENT_BUCKET/$DEPLOYMENT_ZIP $CLUSTERFUZZ_FILE + unzip -q -o $CLUSTERFUZZ_FILE fi # Some configurations (e.g. hosts) run many instances of ClusterFuzz. Don't diff --git a/docker/base/start.sh b/docker/base/start.sh index 64029102035..1ff46017433 100644 --- a/docker/base/start.sh +++ b/docker/base/start.sh @@ -13,6 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +DEPLOYMENT_ZIP="linux-3.zip" +if [[ $CLUSTERFUZZ_RELEASE == "candidate" ]]; then + DEPLOYMENT_ZIP="linux-3-candidate.zip" +fi +export DEPLOYMENT_ZIP + source /data/setup_common.sh source /data/setup_clusterfuzz.sh diff --git a/docker/chromium/base/start.sh b/docker/chromium/base/start.sh index 7c09640cccb..cd49ca88618 100644 --- a/docker/chromium/base/start.sh +++ b/docker/chromium/base/start.sh @@ -13,7 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. + +DEPLOYMENT_ZIP="linux-3.zip" +if [[ $CLUSTERFUZZ_RELEASE == "candidate" ]]; then + DEPLOYMENT_ZIP="linux-3-candidate.zip" +fi + +if [ ! -z $USE_TEST_DEPLOYMENT ]; then + DEPLOYMENT_ZIP="test-deployment/$DEPLOYMENT_ZIP" +fi + +export DEPLOYMENT_ZIP + source /data/setup.sh source /data/setup_clusterfuzz.sh + bash -ex /data/start_clusterfuzz.sh diff --git a/docker/chromium/builder/start.sh b/docker/chromium/builder/start.sh index 9fc2e010f2e..c17641c3e9a 100644 --- a/docker/chromium/builder/start.sh +++ b/docker/chromium/builder/start.sh @@ -13,6 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +DEPLOYMENT_ZIP="linux-3.zip" +if [[ $CLUSTERFUZZ_RELEASE == "candidate" ]]; then + DEPLOYMENT_ZIP="linux-3-candidate.zip" +fi +export DEPLOYMENT_ZIP + source /data/setup_common.sh source /data/setup_depot_tools.sh source /data/setup_gerrit.sh diff --git a/docker/chromium/tests-syncer/start.sh b/docker/chromium/tests-syncer/start.sh index 9fc2e010f2e..c17641c3e9a 100644 --- a/docker/chromium/tests-syncer/start.sh +++ b/docker/chromium/tests-syncer/start.sh @@ -13,6 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +DEPLOYMENT_ZIP="linux-3.zip" +if [[ $CLUSTERFUZZ_RELEASE == "candidate" ]]; then + DEPLOYMENT_ZIP="linux-3-candidate.zip" +fi +export DEPLOYMENT_ZIP + source /data/setup_common.sh source /data/setup_depot_tools.sh source /data/setup_gerrit.sh diff --git a/docker/fuchsia/start.sh b/docker/fuchsia/start.sh index 300f6b65eb7..f46836f782f 100644 --- a/docker/fuchsia/start.sh +++ b/docker/fuchsia/start.sh @@ -13,6 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +DEPLOYMENT_ZIP="linux-3.zip" +if [[ $CLUSTERFUZZ_RELEASE == "candidate" ]]; then + DEPLOYMENT_ZIP="linux-3-candidate.zip" +fi +export DEPLOYMENT_ZIP + + chmod 666 /dev/kvm source /data/setup_common.sh source /data/setup_clusterfuzz.sh diff --git a/docker/oss-fuzz/base/start.sh b/docker/oss-fuzz/base/start.sh index 64029102035..1ff46017433 100644 --- a/docker/oss-fuzz/base/start.sh +++ b/docker/oss-fuzz/base/start.sh @@ -13,6 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +DEPLOYMENT_ZIP="linux-3.zip" +if [[ $CLUSTERFUZZ_RELEASE == "candidate" ]]; then + DEPLOYMENT_ZIP="linux-3-candidate.zip" +fi +export DEPLOYMENT_ZIP + source /data/setup_common.sh source /data/setup_clusterfuzz.sh diff --git a/src/clusterfuzz/_internal/base/utils.py b/src/clusterfuzz/_internal/base/utils.py index 6c00f79bd6a..ecb4befac29 100644 --- a/src/clusterfuzz/_internal/base/utils.py +++ b/src/clusterfuzz/_internal/base/utils.py @@ -227,6 +227,38 @@ def get_application_id(): return app_id +def get_clusterfuzz_release(): + return os.getenv('CLUSTERFUZZ_RELEASE', 'prod') + +def _get_manifest_release_suffix(release): + suffix = '' + if sys.version_info.major == 3: + suffix += '.3' + if release == 'candidate': + suffix += '-candidate' + return suffix + + +def _get_deployment_zip_release_suffix(release): + suffix = '' + if sys.version_info.major == 3: + suffix += '-3' + if release == 'candidate': + suffix += '-candidate' + return suffix + + +def get_platform_deployment_filename(platform, release): + """Return the platform deployment filename.""" + # Expects linux, macos or windows. + base_filename = platform + release_filename_suffix = _get_deployment_zip_release_suffix(release) + return f'{base_filename}{release_filename_suffix}.zip' + + +def get_remote_manifest_filename(release): + return f'clusterfuzz-source.manifest{_get_manifest_release_suffix(release)}' + def service_account_email(): """Get the service account name.""" diff --git a/src/clusterfuzz/_internal/bot/tasks/update_task.py b/src/clusterfuzz/_internal/bot/tasks/update_task.py index 4b87840710d..3a462e66cf0 100644 --- a/src/clusterfuzz/_internal/bot/tasks/update_task.py +++ b/src/clusterfuzz/_internal/bot/tasks/update_task.py @@ -42,8 +42,6 @@ TESTS_LAST_UPDATE_KEY = 'tests_last_update' TESTS_UPDATE_INTERVAL_DAYS = 1 -MANIFEST_FILENAME = 'clusterfuzz-source.manifest.3' - def _rename_dll_for_update(absolute_filepath): """Rename a DLL to allow for updates.""" @@ -51,18 +49,6 @@ def _rename_dll_for_update(absolute_filepath): os.rename(absolute_filepath, backup_filepath) -def _platform_deployment_filename(): - """Return the platform deployment filename.""" - platform_mappings = { - 'Linux': 'linux', - 'Windows': 'windows', - 'Darwin': 'macos' - } - - base_filename = platform_mappings[platform.system()] + '-3' - return base_filename + '.zip' - - def _deployment_file_url(filename): """Helper to return deployment file url.""" deployment_bucket = local_config.ProjectConfig().get('deployment.bucket') @@ -76,12 +62,21 @@ def _deployment_file_url(filename): def get_source_url(): """Return the source URL.""" - return _deployment_file_url(_platform_deployment_filename()) + release = utils.get_clusterfuzz_release() + platform_name = platform.system() + platform_mappings = { + 'Linux': 'linux', + 'Windows': 'windows', + 'Darwin': 'macos' + } + platform_name= platform_mappings[platform_name] + return _deployment_file_url(utils.get_platform_deployment_filename(platform_name, release)) def get_source_manifest_url(): """Return the source manifest URL.""" - return _deployment_file_url(MANIFEST_FILENAME) + release = utils.get_clusterfuzz_release() + return _deployment_file_url(utils.get_remote_manifest_filename(release)) def clear_old_files(directory, extracted_file_set): @@ -156,14 +151,14 @@ def get_newer_source_revision(): logs.info('No manifest found. Forcing an update.') return source_version - logs.info('Local source code version: %s.' % local_source_version) - logs.info('Remote source code version: %s.' % source_version) + logs.info(f'Local source code version: {local_source_version}, on release {utils.get_clusterfuzz_release()}.') + logs.info(f'Remote source code version: {source_version}, on release {utils.get_clusterfuzz_release()}.') if local_source_version >= source_version: logs.info('Remote souce code <= local source code. No update.') # No source code update found. Source code is current, bail out. return None - logs.info(f'New source code: {source_version}') + logs.info(f'New source code: {source_version} (updated from {local_source_version}, on release {utils.get_clusterfuzz_release()})') return source_version @@ -276,7 +271,7 @@ def update_source_code(): source_version = utils.read_data_from_file( local_manifest_path, eval_data=False).decode('utf-8').strip() os.remove(temp_archive) - logs.info('Source code updated to %s.' % source_version) + logs.info(f'Source code updated to {source_version} (release = {utils.get_clusterfuzz_release()}).') def update_tests_if_needed(): diff --git a/src/clusterfuzz/_internal/metrics/logs.py b/src/clusterfuzz/_internal/metrics/logs.py index 7e3b92a4ab5..6a69e8fb5cb 100644 --- a/src/clusterfuzz/_internal/metrics/logs.py +++ b/src/clusterfuzz/_internal/metrics/logs.py @@ -553,7 +553,9 @@ def emit(level, message, exc_info=None, **extras): 'path': path_name, 'line': line_number, 'method': method_name - } + }, + 'release': os.environ.get('CLUSTERFUZZ_RELEASE', 'prod'), + 'docker_image': os.environ.get('DOCKER_IMAGE', '') }) diff --git a/src/clusterfuzz/_internal/tests/core/metrics/logs_test.py b/src/clusterfuzz/_internal/tests/core/metrics/logs_test.py index 3fd688dbd76..e8aa8d70457 100644 --- a/src/clusterfuzz/_internal/tests/core/metrics/logs_test.py +++ b/src/clusterfuzz/_internal/tests/core/metrics/logs_test.py @@ -315,6 +315,8 @@ def test_info(self): 'target': 'bot', 'test': 'yes' }, + 'release': 'prod', + 'docker_image': '', 'location': { 'path': os.path.abspath(__file__).rstrip('c'), 'line': statement_line, @@ -339,6 +341,8 @@ def test_error(self): 'target': 'bot', 'test': 'yes' }, + 'release': 'prod', + 'docker_image': '', 'location': { 'path': os.path.abspath(__file__).rstrip('c'), 'line': statement_line, diff --git a/src/local/butler/deploy.py b/src/local/butler/deploy.py index 32d14bbb6d6..05102b5aceb 100644 --- a/src/local/butler/deploy.py +++ b/src/local/butler/deploy.py @@ -27,6 +27,7 @@ from local.butler import common from local.butler import constants from local.butler import package +from clusterfuzz._internal.base import utils from src.clusterfuzz._internal.config import local_config from src.clusterfuzz._internal.system import environment @@ -92,7 +93,8 @@ def _deploy_app_prod(project, yaml_paths, package_zip_paths, deploy_appengine=True, - test_deployment=False): + test_deployment=False, + release='prod'): """Deploy app in production.""" if deploy_appengine: services = _get_services(yaml_paths) @@ -116,7 +118,8 @@ def _deploy_app_prod(project, _deploy_manifest( deployment_bucket, constants.PACKAGE_TARGET_MANIFEST_PATH, - test_deployment=test_deployment) + test_deployment=test_deployment, + release=release) def _deploy_app_staging(project, yaml_paths): @@ -227,16 +230,21 @@ def _deploy_zip(bucket_name, zip_path, test_deployment=False): os.path.basename(zip_path))) -def _deploy_manifest(bucket_name, manifest_path, test_deployment=False): +def _deploy_manifest(bucket_name, + manifest_path, + test_deployment=False, + release='prod'): """Deploy source manifest to GCS.""" + remote_manifest_path = utils.get_remote_manifest_filename(release) + if test_deployment: common.execute(f'gsutil cp {manifest_path} ' f'gs://{bucket_name}/test-deployment/' - f'clusterfuzz-source.manifest.3') + f'{remote_manifest_path}') else: common.execute(f'gsutil cp {manifest_path} ' f'gs://{bucket_name}/' - f'clusterfuzz-source.manifest.3') + f'{remote_manifest_path}') def _update_deployment_manager(project, name, config_path): @@ -390,7 +398,8 @@ def _prod_deployment_helper(config_dir, package_zip_paths, deploy_appengine=True, deploy_k8s=True, - test_deployment=False): + test_deployment=False, + release='prod'): """Helper for production deployment.""" config = local_config.Config() deployment_bucket = config.get('project.deployment.bucket') @@ -418,7 +427,8 @@ def _prod_deployment_helper(config_dir, yaml_paths, package_zip_paths, deploy_appengine=deploy_appengine, - test_deployment=test_deployment) + test_deployment=test_deployment, + release=release) if deploy_appengine: common.execute( @@ -529,7 +539,8 @@ def execute(args): if deploy_zips: for platform_name in platforms: package_zip_paths.append( - package.package(revision, platform_name=platform_name)) + package.package( + revision, platform_name=platform_name, release=args.release)) else: # package.package calls these, so only set these up if we're not packaging, # since they can be fairly slow. @@ -553,7 +564,8 @@ def execute(args): package_zip_paths, deploy_appengine, deploy_k8s, - test_deployment=test_deployment) + test_deployment=test_deployment, + release=args.release) with open(constants.PACKAGE_TARGET_MANIFEST_PATH) as f: print('Source updated to %s' % f.read()) diff --git a/src/local/butler/package.py b/src/local/butler/package.py index 5824614aece..5a1cf79f194 100644 --- a/src/local/butler/package.py +++ b/src/local/butler/package.py @@ -18,6 +18,7 @@ import sys import zipfile +from clusterfuzz._internal.base import utils from local.butler import appengine from local.butler import common from local.butler import constants @@ -70,7 +71,8 @@ def _get_files(path): def package(revision, target_zip_dir=constants.PACKAGE_TARGET_ZIP_DIRECTORY, target_manifest_path=constants.PACKAGE_TARGET_MANIFEST_PATH, - platform_name=None): + platform_name=None, + release='prod'): """Prepare clusterfuzz-source.zip.""" is_ci = os.getenv('TEST_BOT_ENVIRONMENT') if not is_ci and common.is_git_dirty(): @@ -95,7 +97,7 @@ def package(revision, target_zip_name = constants.LEGACY_ZIP_NAME if platform_name: - target_zip_name = platform_name + '-3.zip' + target_zip_name = utils.get_platform_deployment_filename(platform_name, release) target_zip_path = os.path.join(target_zip_dir, target_zip_name) _clear_zip(target_zip_path) @@ -136,11 +138,15 @@ def package(revision, def execute(args): + """Execute the butler package command.""" if args.platform == 'all': for platform_name in list(constants.PLATFORMS.keys()): package( revision=common.compute_staging_revision(), - platform_name=platform_name) + platform_name=platform_name, + release=args.release) else: package( - revision=common.compute_staging_revision(), platform_name=args.platform) + revision=common.compute_staging_revision(), + platform_name=args.platform, + release=args.release) diff --git a/src/local/remote/handlers/posix.py b/src/local/remote/handlers/posix.py index 6887b52c09c..8da4341f254 100644 --- a/src/local/remote/handlers/posix.py +++ b/src/local/remote/handlers/posix.py @@ -130,7 +130,8 @@ def stage(self, config_dir): local_zip_path = package.package( revision=butler_common.compute_staging_revision(), - platform_name=self.platform) + platform_name=self.platform, + release='prod') self._copy_staging_archive_from_local_to_remote(local_zip_path) self._run(f'cd {self.clusterfuzz_parent_path} && ' diff --git a/src/local/remote/handlers/windows.py b/src/local/remote/handlers/windows.py index 4fef18e258c..53e81221ce2 100644 --- a/src/local/remote/handlers/windows.py +++ b/src/local/remote/handlers/windows.py @@ -119,7 +119,8 @@ def stage(self, config_dir): zip_path = package.package( revision=butler_common.compute_staging_revision(), - platform_name='windows') + platform_name='windows', + release='prod') remote_zip_path = ( f'{self.clusterfuzz_parent_path}\\{self.staging_source_filename}') api.put(zip_path, remote_zip_path)