[Python 3.11 Update] Primitive progressive rollout implementation

butler.py

@@ -259,6 +259,7 @@ def main():
       'package', help='Package clusterfuzz with a staging revision')
   parser_package.add_argument(
       '-p', '--platform', choices=['linux', 'macos', 'windows', 'all'])
+  parser_package.add_argument('-r', '--release', choices=['prod', 'candidate'], default='prod')
 
   parser_deploy = subparsers.add_parser('deploy', help='Deploy to Appengine')
   parser_deploy.add_argument(
@@ -274,6 +275,7 @@ def main():
       '--prod', action='store_true', help='Deploy to production.')
   parser_deploy.add_argument(
       '--targets', nargs='*', default=['appengine', 'k8s', 'zips'])
+  parser_deploy.add_argument('--release', '-r', choices=['prod', 'candidate'], default='prod')
 
   parser_run_server = subparsers.add_parser(
       'run_server', help='Run the local Clusterfuzz server.')

docker/base/Dockerfile

@@ -97,7 +97,7 @@ RUN wget https://download.java.net/java/GA/jdk15.0.2/0d1cfde4252546c6931946de8db
     tar -xzv --strip-components=1 -f openjdk-15.0.2_linux-x64_bin.tar.gz --directory $JAVA_HOME && \
     rm -rf openjdk*.tar.gz $JAVA_HOME/jmods $JAVA_HOME/lib/src.zip
 
-# Install Python 3.
+# Install Python 3.7
 RUN curl -sS https://www.python.org/ftp/python/3.7.7/Python-3.7.7.tgz | tar -C /tmp -xzv && \
     cd /tmp/Python-3.7.7 && \
     ./configure --enable-optimizations --enable-loadable-sqlite-extensions && make altinstall && \
@@ -131,7 +131,6 @@ ENV ROOT_DIR $INSTALL_DIRECTORY/clusterfuzz
 ENV UPDATE_WEB_TESTS True
 ENV PYTHONPATH $INSTALL_DIRECTORY/clusterfuzz/src
 ENV RUN_CMD "python3.7 $ROOT_DIR/src/python/bot/startup/run.py"
-ENV DEPLOYMENT_ZIP "linux-3.zip"
 
 # Passwordless sudo (needed for AFL launcher).
 RUN groupadd nopwsudo && \

docker/base/setup_clusterfuzz.sh

@@ -18,15 +18,17 @@ if [ -z "$DEPLOYMENT_BUCKET" ]; then
   export DEPLOYMENT_BUCKET=$(curl -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/project/attributes/deployment-bucket)
 fi
 
+CLUSTERFUZZ_FILE=clusterfuzz_package.zip
 # When $LOCAL_SRC is set, use source zip on mounted volume for local testing.
 if [[ -z "$LOCAL_SRC" ]]; then
   # Set up ClusterFuzz
   if [[ -d clusterfuzz ]]; then
     rm -rf clusterfuzz
   fi
 
-  gsutil cp gs://$DEPLOYMENT_BUCKET/$DEPLOYMENT_ZIP .
-  unzip -q -o $DEPLOYMENT_ZIP
+  # DEPLOYMENT_ZIP might be test-deployment/linux-3.zip, so we do not extract DEPLOYMENT_ZIP directly
+  gsutil cp gs://$DEPLOYMENT_BUCKET/$DEPLOYMENT_ZIP $CLUSTERFUZZ_FILE
+  unzip -q -o $CLUSTERFUZZ_FILE
 fi
 
 # Some configurations (e.g. hosts) run many instances of ClusterFuzz. Don't

docker/base/start.sh

@@ -13,6 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+DEPLOYMENT_ZIP="linux-3.zip"
+if [[ $CLUSTERFUZZ_RELEASE == "candidate" ]]; then
+    DEPLOYMENT_ZIP="linux-3-candidate.zip"
+fi
+export DEPLOYMENT_ZIP
+
 source /data/setup_common.sh
 source /data/setup_clusterfuzz.sh
 

docker/chromium/base/start.sh

@@ -13,7 +13,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+
+DEPLOYMENT_ZIP="linux-3.zip"
+if [[ $CLUSTERFUZZ_RELEASE == "candidate" ]]; then
+    DEPLOYMENT_ZIP="linux-3-candidate.zip"
+fi
+
+if [ ! -z $USE_TEST_DEPLOYMENT ]; then
+    DEPLOYMENT_ZIP="test-deployment/$DEPLOYMENT_ZIP"
+fi
+
+export DEPLOYMENT_ZIP
+
 source /data/setup.sh
 source /data/setup_clusterfuzz.sh
 
+
 bash -ex /data/start_clusterfuzz.sh

docker/chromium/builder/start.sh

@@ -13,6 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+DEPLOYMENT_ZIP="linux-3.zip"
+if [[ $CLUSTERFUZZ_RELEASE == "candidate" ]]; then
+    DEPLOYMENT_ZIP="linux-3-candidate.zip"
+fi
+export DEPLOYMENT_ZIP
+
 source /data/setup_common.sh
 source /data/setup_depot_tools.sh
 source /data/setup_gerrit.sh

docker/chromium/tests-syncer/start.sh

@@ -13,6 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+DEPLOYMENT_ZIP="linux-3.zip"
+if [[ $CLUSTERFUZZ_RELEASE == "candidate" ]]; then
+    DEPLOYMENT_ZIP="linux-3-candidate.zip"
+fi
+export DEPLOYMENT_ZIP
+
 source /data/setup_common.sh
 source /data/setup_depot_tools.sh
 source /data/setup_gerrit.sh

docker/fuchsia/start.sh

@@ -13,6 +13,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+DEPLOYMENT_ZIP="linux-3.zip"
+if [[ $CLUSTERFUZZ_RELEASE == "candidate" ]]; then
+    DEPLOYMENT_ZIP="linux-3-candidate.zip"
+fi
+export DEPLOYMENT_ZIP
+
+
 chmod 666 /dev/kvm
 source /data/setup_common.sh
 source /data/setup_clusterfuzz.sh

docker/oss-fuzz/base/start.sh

@@ -13,6 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+DEPLOYMENT_ZIP="linux-3.zip"
+if [[ $CLUSTERFUZZ_RELEASE == "candidate" ]]; then
+    DEPLOYMENT_ZIP="linux-3-candidate.zip"
+fi
+export DEPLOYMENT_ZIP
+
 source /data/setup_common.sh
 source /data/setup_clusterfuzz.sh
 

src/clusterfuzz/_internal/base/utils.py

@@ -227,6 +227,38 @@ def get_application_id():
 
   return app_id
 
+def get_clusterfuzz_release():
+  return os.getenv('CLUSTERFUZZ_RELEASE', 'prod')
+
+def _get_manifest_release_suffix(release):
+  suffix = ''
+  if sys.version_info.major == 3:
+    suffix += '.3'
+  if release == 'candidate':
+    suffix += '-candidate'
+  return suffix
+
+
+def _get_deployment_zip_release_suffix(release):
+  suffix = ''
+  if sys.version_info.major == 3:
+    suffix += '-3'
+  if release == 'candidate':
+    suffix += '-candidate'
+  return suffix
+
+
+def get_platform_deployment_filename(platform, release):
+  """Return the platform deployment filename."""
+  # Expects linux, macos or windows.
+  base_filename = platform 
+  release_filename_suffix = _get_deployment_zip_release_suffix(release)
+  return f'{base_filename}{release_filename_suffix}.zip'
+
+
+def get_remote_manifest_filename(release):
+  return f'clusterfuzz-source.manifest{_get_manifest_release_suffix(release)}'
+
 
 def service_account_email():
   """Get the service account name."""

src/clusterfuzz/_internal/bot/tasks/update_task.py

@@ -42,27 +42,13 @@
 TESTS_LAST_UPDATE_KEY = 'tests_last_update'
 TESTS_UPDATE_INTERVAL_DAYS = 1
 
-MANIFEST_FILENAME = 'clusterfuzz-source.manifest.3'
-
 
 def _rename_dll_for_update(absolute_filepath):
   """Rename a DLL to allow for updates."""
   backup_filepath = absolute_filepath + '.bak.' + str(int(time.time()))
   os.rename(absolute_filepath, backup_filepath)
 
 
-def _platform_deployment_filename():
-  """Return the platform deployment filename."""
-  platform_mappings = {
-      'Linux': 'linux',
-      'Windows': 'windows',
-      'Darwin': 'macos'
-  }
-
-  base_filename = platform_mappings[platform.system()] + '-3'
-  return base_filename + '.zip'
-
-
 def _deployment_file_url(filename):
   """Helper to return deployment file url."""
   deployment_bucket = local_config.ProjectConfig().get('deployment.bucket')
@@ -76,12 +62,21 @@ def _deployment_file_url(filename):
 
 def get_source_url():
   """Return the source URL."""
-  return _deployment_file_url(_platform_deployment_filename())
+  release = utils.get_clusterfuzz_release()
+  platform_name = platform.system()
+  platform_mappings = {
+      'Linux': 'linux',
+      'Windows': 'windows',
+      'Darwin': 'macos'
+  }
+  platform_name= platform_mappings[platform_name]
+  return _deployment_file_url(utils.get_platform_deployment_filename(platform_name, release))
 
 
 def get_source_manifest_url():
   """Return the source manifest URL."""
-  return _deployment_file_url(MANIFEST_FILENAME)
+  release = utils.get_clusterfuzz_release()
+  return _deployment_file_url(utils.get_remote_manifest_filename(release))
 
 
 def clear_old_files(directory, extracted_file_set):
@@ -156,14 +151,14 @@ def get_newer_source_revision():
     logs.info('No manifest found. Forcing an update.')
     return source_version
 
-  logs.info('Local source code version: %s.' % local_source_version)
-  logs.info('Remote source code version: %s.' % source_version)
+  logs.info(f'Local source code version: {local_source_version}, on release {utils.get_clusterfuzz_release()}.')
+  logs.info(f'Remote source code version: {source_version}, on release {utils.get_clusterfuzz_release()}.')
   if local_source_version >= source_version:
     logs.info('Remote souce code <= local source code. No update.')
     # No source code update found. Source code is current, bail out.
     return None
 
-  logs.info(f'New source code: {source_version}')
+  logs.info(f'New source code: {source_version} (updated from {local_source_version}, on release {utils.get_clusterfuzz_release()})')
   return source_version
 
 
@@ -276,7 +271,7 @@ def update_source_code():
   source_version = utils.read_data_from_file(
       local_manifest_path, eval_data=False).decode('utf-8').strip()
   os.remove(temp_archive)
-  logs.info('Source code updated to %s.' % source_version)
+  logs.info(f'Source code updated to {source_version} (release = {utils.get_clusterfuzz_release()}).')
 
 
 def update_tests_if_needed():

src/clusterfuzz/_internal/metrics/logs.py

@@ -553,7 +553,9 @@ def emit(level, message, exc_info=None, **extras):
               'path': path_name,
               'line': line_number,
               'method': method_name
-          }
+          },
+          'release': os.environ.get('CLUSTERFUZZ_RELEASE', 'prod'),
+          'docker_image': os.environ.get('DOCKER_IMAGE', '')
       })
 
 

src/clusterfuzz/_internal/tests/core/metrics/logs_test.py

@@ -315,6 +315,8 @@ def test_info(self):
                 'target': 'bot',
                 'test': 'yes'
             },
+            'release': 'prod',
+            'docker_image': '',
             'location': {
                 'path': os.path.abspath(__file__).rstrip('c'),
                 'line': statement_line,
@@ -339,6 +341,8 @@ def test_error(self):
                 'target': 'bot',
                 'test': 'yes'
             },
+            'release': 'prod',
+            'docker_image': '',
             'location': {
                 'path': os.path.abspath(__file__).rstrip('c'),
                 'line': statement_line,

src/local/butler/deploy.py

@@ -27,6 +27,7 @@
 from local.butler import common
 from local.butler import constants
 from local.butler import package
+from clusterfuzz._internal.base import utils
 from src.clusterfuzz._internal.config import local_config
 from src.clusterfuzz._internal.system import environment
 
@@ -92,7 +93,8 @@ def _deploy_app_prod(project,
                      yaml_paths,
                      package_zip_paths,
                      deploy_appengine=True,
-                     test_deployment=False):
+                     test_deployment=False,
+                     release='prod'):
   """Deploy app in production."""
   if deploy_appengine:
     services = _get_services(yaml_paths)
@@ -116,7 +118,8 @@ def _deploy_app_prod(project,
     _deploy_manifest(
         deployment_bucket,
         constants.PACKAGE_TARGET_MANIFEST_PATH,
-        test_deployment=test_deployment)
+        test_deployment=test_deployment,
+        release=release)
 
 
 def _deploy_app_staging(project, yaml_paths):
@@ -227,16 +230,21 @@ def _deploy_zip(bucket_name, zip_path, test_deployment=False):
                                                 os.path.basename(zip_path)))
 
 
-def _deploy_manifest(bucket_name, manifest_path, test_deployment=False):
+def _deploy_manifest(bucket_name,
+                     manifest_path,
+                     test_deployment=False,
+                     release='prod'):
   """Deploy source manifest to GCS."""
+  remote_manifest_path = utils.get_remote_manifest_filename(release)
+
   if test_deployment:
     common.execute(f'gsutil cp {manifest_path} '
                    f'gs://{bucket_name}/test-deployment/'
-                   f'clusterfuzz-source.manifest.3')
+                   f'{remote_manifest_path}')
   else:
     common.execute(f'gsutil cp {manifest_path} '
                    f'gs://{bucket_name}/'
-                   f'clusterfuzz-source.manifest.3')
+                   f'{remote_manifest_path}')
 
 
 def _update_deployment_manager(project, name, config_path):
@@ -390,7 +398,8 @@ def _prod_deployment_helper(config_dir,
                             package_zip_paths,
                             deploy_appengine=True,
                             deploy_k8s=True,
-                            test_deployment=False):
+                            test_deployment=False,
+                            release='prod'):
   """Helper for production deployment."""
   config = local_config.Config()
   deployment_bucket = config.get('project.deployment.bucket')
@@ -418,7 +427,8 @@ def _prod_deployment_helper(config_dir,
       yaml_paths,
       package_zip_paths,
       deploy_appengine=deploy_appengine,
-      test_deployment=test_deployment)
+      test_deployment=test_deployment,
+      release=release)
 
   if deploy_appengine:
     common.execute(
@@ -529,7 +539,8 @@ def execute(args):
   if deploy_zips:
     for platform_name in platforms:
       package_zip_paths.append(
-          package.package(revision, platform_name=platform_name))
+          package.package(
+              revision, platform_name=platform_name, release=args.release))
   else:
     # package.package calls these, so only set these up if we're not packaging,
     # since they can be fairly slow.
@@ -553,7 +564,8 @@ def execute(args):
         package_zip_paths,
         deploy_appengine,
         deploy_k8s,
-        test_deployment=test_deployment)
+        test_deployment=test_deployment,
+        release=args.release)
 
   with open(constants.PACKAGE_TARGET_MANIFEST_PATH) as f:
     print('Source updated to %s' % f.read())