Merge branch 'google:master' into patch-1

Author: GameRoMan

.gitignore

@@ -22,6 +22,7 @@
 /bot/*
 /coverage
 /deployment
+/clusterfuzz-config
 /local/bin/android-sdk/*
 /local/storage
 /paramiko.log

.tool-versions

@@ -0,0 +1,2 @@
+python 3.11.9
+gcloud 523.0.1

AGENTS.md

@@ -0,0 +1,86 @@
+# Agent Instructions
+
+This document provides instructions for agents on how to perform common development tasks in this repository.
+
+## Initial Setup
+
+This project uses `asdf` to manage tool versions. Before you can install the project's dependencies, you need to install and configure `asdf`.
+
+### 1. Install `asdf`
+
+First, make sure you have `git` and `curl` installed. Then, clone the `asdf` repository:
+
+```bash
+git clone https://github.com/asdf-vm/asdf.git ~/.asdf --branch v0.14.0
+```
+
+### 2. Configure Your Shell
+
+Add `asdf` to your shell's startup file. For `bash`, run:
+
+```bash
+echo -e "\n. \"$HOME/.asdf/asdf.sh\"" >> ~/.bashrc
+```
+
+Then, source your `.bashrc` to apply the changes to your current session:
+
+```bash
+source ~/.bashrc
+```
+
+### 3. Install `asdf` Plugins and Tools
+
+Now, from the root of this project, run the following commands to install the necessary `asdf` plugins and the tools specified in the `.tool-versions` file:
+
+```bash
+asdf plugin add python
+asdf plugin add gcloud https://github.com/jthegedus/asdf-gcloud
+asdf install
+```
+
+### 4. Install `pipenv`
+
+Once `asdf` has installed the correct python version, you need to install `pipenv`:
+
+```bash
+python -m pip install pipenv
+```
+
+### 5. Install Project Dependencies
+
+Now you are ready to install the project's dependencies. Run the following command from the root of the repository:
+
+```bash
+./local/install_deps.bash
+```
+
+## Testing
+
+To run all unit tests, execute the following commands:
+
+```bash
+python butler.py py_unittest -t appengine -m
+python butler.py py_unittest -t core -m
+```
+
+The `-m` flag runs the tests in parallel, which is recommended.
+
+## Linting
+
+To check the code for style and linting issues, run the following command:
+
+```bash
+python butler.py lint
+```
+
+This will lint the changed code in your current branch.
+
+## Formatting
+
+To automatically format the code to match the project's style guidelines, run:
+
+```bash
+python butler.py format
+```
+
+This will format the changed code in your current branch.

butler.py

@@ -304,7 +304,10 @@ def main():
   parser_deploy.add_argument(
       '--prod', action='store_true', help='Deploy to production.')
   parser_deploy.add_argument(
-      '--targets', nargs='*', default=['appengine', 'terraform', 'zips'])
+      '--targets',
+      nargs='*',
+      default=['appengine', 'terraform', 'zips'],
+      choices=['appengine', 'terraform', 'zips', 'test_deployment'])
   parser_deploy.add_argument(
       '--release',
       '-r',

configs/test/gae/prod3/cron.yaml

@@ -70,11 +70,6 @@ cron:
     schedule: every day 01:00
     target: cron-service
 
-  - description: Manage VMs.
-    url: /manage-vms
-    schedule: every 30 minutes
-    target: cron-service
-
   - description: Schedule corpus pruning task.
     url: /schedule-corpus-pruning
     schedule: every day 19:00

configs/test/k8s/manage-vms.yaml

@@ -139,3 +139,6 @@ env:
 # Pub/Sub topic for receiving issue updates.
 #issue_updates:
   #pubsub_topic: projects/test-project/topics/issue-updates
+
+schedule_fuzz:
+  cpu_limit: 100000

configs/test/project.yaml

@@ -0,0 +1,100 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+steps:
+- name: gcr.io/cloud-builders/git
+  entrypoint: bash
+  args:
+  - -c
+  - |
+    # First, clone the repository
+    git config --global credential.helper gcloud.sh
+    git clone https://clusterfuzz-config-472119376969-git.us-central1.sourcemanager.dev/clusterfuzz-testing/clusterfuzz-config.git /workspace/clusterfuzz-config
+    
+    # Move into the repository directory
+    cd /workspace/clusterfuzz-config
+    
+    # Conditionally run 'git checkout'
+    if [ -n "${_CLUSTERFUZZ_CONFIG_REVISION}" ]; then
+      echo "✅ _CLUSTERFUZZ_CONFIG_REVISION is set. Checking out to commit: ${_CLUSTERFUZZ_CONFIG_REVISION}"
+      git checkout "${_CLUSTERFUZZ_CONFIG_REVISION}"
+    else
+      echo "☑️ _CLUSTERFUZZ_CONFIG_REVISION is not set. Using the latest commit."
+    fi
+
+- name: gcr.io/cloud-builders/git
+  entrypoint: bash
+  args:
+  - -c
+  - |
+    # Conditionally clone the upstream clusterfuzz and 'git checkout' if the revision is provided
+    if [ -n "${_CLUSTERFUZZ_REVISION}" ]; then
+      echo "✅ _CLUSTERFUZZ_REVISION is set. Checking out to commit: ${_CLUSTERFUZZ_REVISION}"
+      git clone https://github.com/google/clusterfuzz.git
+      cd clusterfuzz
+      git checkout "${_CLUSTERFUZZ_REVISION}"
+    else
+      echo "☑️ _CLUSTERFUZZ_REVISION is not set. Using the latest commit."
+    fi
+
+- name: gcr.io/clusterfuzz-images/base:091c6c2-202409251610
+  entrypoint: bash
+  args:
+  - -c
+  - |
+    # Conditionally run go into the cloned clusterfuzz
+    if [ -n "${_CLUSTERFUZZ_REVISION}" ]; then
+      cd clusterfuzz
+    fi
+    
+    # Install required deps for performing butler deploy
+    bash ./local/install_deps.bash
+    
+    # Get into the venv
+    source "$$(python3.11 -m pipenv --venv)/bin/activate"
+
+    # Check if the _PROJECT substitution was provided.
+    if [ -z "${_PROJECT}" ]; then
+      echo "❌ Error: The _PROJECT substitution variable must be provided."
+      exit 1
+    fi
+
+    # Construct the full path using the substitution variable.
+    configpath="/workspace/clusterfuzz-config/configs/${_PROJECT}"
+
+    # Verify that the configuration directory actually exists.
+    if [ ! -d "$configpath" ]; then
+      echo "❌ Error: Configuration directory not found at $configpath"
+      exit 1
+    fi
+
+    echo "✅ Running deploy for project config: $configpath"
+    python3.11 butler.py deploy -c $configpath --prod --targets zips appengine --force
+
+- name: gcr.io/google.com/cloudsdktool/cloud-sdk
+  entrypoint: bash
+  args:
+  - -c
+  - |
+    if [ -n "${_CLUSTERFUZZ_REVISION}" ]; then
+      cd clusterfuzz
+    fi
+    CURRENT_CLUSTERFUZZ_REVISION="$(cat src/appengine/resources/clusterfuzz-source.manifest)"
+    gcloud compute project-info add-metadata --metadata=clusterfuzz-revision="$${CURRENT_CLUSTERFUZZ_REVISION}" --project "${PROJECT_ID}"
+
+timeout: 1200s
+options:
+  machineType: E2_HIGHCPU_32
+  diskSizeGb: 500
+  logging: CLOUD_LOGGING_ONLY

deploy/cloudbuild.yaml

@@ -205,13 +205,13 @@ GEM
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
     mercenary (0.3.6)
-    mini_portile2 (2.8.8)
+    mini_portile2 (2.8.9)
     minima (2.5.1)
       jekyll (>= 3.5, < 5.0)
       jekyll-feed (~> 0.9)
       jekyll-seo-tag (~> 2.1)
     minitest (5.19.0)
-    nokogiri (1.18.8)
+    nokogiri (1.18.9)
       mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
     octokit (4.25.1)

docs/Gemfile.lock

@@ -184,13 +184,9 @@ to deploy new changes.
 
 ### Google Compute Engine bots
 
-Once deployed, bots are automatically created via an App Engine cron task that
-runs every *30 minutes*. For faster results, you can manually force it by
-visiting the `Cron jobs` page
-[here](https://console.cloud.google.com/appengine/cronjobs) and running the
-`/manage-vms` cron job.
+ClusterFuzz bots must be created (preferably by Terraform) following the configurations in configs/test/gce/clusters.yaml.
 
-After the cron job finishes, check this
+After creation or update finishes, check
 [page](https://console.cloud.google.com/compute/instanceGroups/list) to ensure
 that all instances are created for this particular
 [instance group](#google-compute-engine-cluster).