chore: Change all the references of OwlBot.yaml to OwlBot-hermetic.ya…

…ml. (#2642) This is follow up of googleapis/google-cloud-java#10696. The hermetic build generation process would fail without this PR due to unable to find OwlBot.yaml files. This PR changes all the references of `OwlBot.yaml` to `OwlBot-hermetic.yaml`, removes `OwlBot.yaml` from a few places that don't need to reference it in the first places.
googleapis · Apr 16, 2024 · e542af8 · e542af8
1 parent 2c93542
commit e542af8
Show file tree

Hide file tree

Showing 10 changed files with 44 additions and 53 deletions.
diff --git a/.gitignore b/.gitignore
@@ -24,7 +24,6 @@ target/
 # library generation
 **/output/
 **/googleapis
-library_generation/test/**/golden*/
 library_generation/test/resources/test_monorepo_postprocessing/
 **/*egg-info/
 **/build/

diff --git a/library_generation/README.md b/library_generation/README.md
@@ -55,24 +55,24 @@ of versions.txt.
 For each module (e.g. `google-cloud-java/java-asset`), the following files/folders
 will be created/modified:
 
-| Name                        | Notes                                                                    |
-|:----------------------------|:-------------------------------------------------------------------------|
-| google-*/                   | Source code generated by gapic-generator-java                            |
-| google-*/pom.xml            | Only be generated if it does not exist                                   |
-| grpc-*/                     | Source code generated by grpc generator, one per each version            |
-| grpc-*/pom.xml              | Only be generated if it does not exist                                   |
-| proto-*/                    | Source code generated by Protobuf default compiler, one per each version |
-| proto-*/pom.xml             | Only be generated if it does not exist                                   |
-| samples/snippets/generated/ | Only be generated if `include_samples` is set to true                    |
-| google-*-bom/pom.xml        | Library BOM, only be generated if it does not exist                      |
-| pom.xml                     | Library parent BOM, only be generated if it does not exist               |
-| .repo-metadata.json         | Always generated from inputs                                             |
-| .OwlBot.yaml                | Only be generated from a template if it does not exist                   |
-| owlbot.py                   | Only be generated from a template if it does not exist                   |
-| README.md                   | Always generated from inputs                                             |
-| gapic-libraries-bom/pom.xml | Always generated from inputs                                             |
-| pom.xml (repo root dir)     | Always generated from inputs                                             |
-| versions.txt                | New entries will be added if they don’t exist                            |
+| Name                                | Notes                                                                    |
+|:------------------------------------|:-------------------------------------------------------------------------|
+| google-*/                           | Source code generated by gapic-generator-java                            |
+| google-*/pom.xml                    | Only be generated if it does not exist                                   |
+| grpc-*/                             | Source code generated by grpc generator, one per each version            |
+| grpc-*/pom.xml                      | Only be generated if it does not exist                                   |
+| proto-*/                            | Source code generated by Protobuf default compiler, one per each version |
+| proto-*/pom.xml                     | Only be generated if it does not exist                                   |
+| samples/snippets/generated/         | Only be generated if `include_samples` is set to true                    |
+| google-*-bom/pom.xml                | Library BOM, only be generated if it does not exist                      |
+| pom.xml                             | Library parent BOM, only be generated if it does not exist               |
+| .repo-metadata.json                 | Always generated from inputs                                             |
+| .OwlBot-hermetic.yaml               | Only be generated from a template if it does not exist                   |
+| owlbot.py                           | Only be generated from a template if it does not exist                   |
+| README.md                           | Always generated from inputs                                             |
+| gapic-libraries-bom/pom.xml         | Always generated from inputs                                             |
+| pom.xml (repo root dir)             | Always generated from inputs                                             |
+| versions.txt                        | New entries will be added if they don’t exist                            |
 
 ### googleapis commit history
 
@@ -231,7 +231,7 @@ $repository_path
 |    |_samples
 |    |    |_snippets
 |    |    |    |_generated
-|    |_.OwlBot.yaml
+|    |_.OwlBot-hermetic.yaml
 |    |_.repo-metadata.json
 |    |_owlbot.py
 |    |_pom.xml
@@ -275,7 +275,7 @@ $repository_path
 |    |_samples
 |    |    |_snippets
 |    |    |    |_generated
-|    |_.OwlBot.yaml
+|    |_.OwlBot-hermetic.yaml
 |    |_.repo-metadata.json
 |    |_owlbot.py
 |    |_pom.xml

diff --git a/library_generation/generate_composed_library.py b/library_generation/generate_composed_library.py
@@ -72,7 +72,7 @@ def generate_composed_library(
         build_file_folder = Path(f"{output_folder}/{gapic.proto_path}").resolve()
         print(f"build_file_folder: {build_file_folder}")
         gapic_inputs = parse_build_file(build_file_folder, gapic.proto_path)
-        # generate prerequisite files (.repo-metadata.json, .OwlBot.yaml,
+        # generate prerequisite files (.repo-metadata.json, .OwlBot-hermetic.yaml,
         # owlbot.py) here because transport is parsed from BUILD.bazel,
         # which lives in a versioned proto_path.
         util.generate_prerequisite_files(

diff --git a/library_generation/owlbot/bin/entrypoint.sh b/library_generation/owlbot/bin/entrypoint.sh
@@ -27,19 +27,10 @@ set -ex
 scripts_root=$1
 versions_file=$2
 configuration_yaml=$3
+is_monorepo=$4
 
 
-# This script can be used to process HW libraries and monorepo
-# (google-cloud-java) libraries, which require a slightly different treatment
-# monorepo folders have an .OwlBot.yaml file in the module folder (e.g.
-# java-asset/.OwlBot.yaml), whereas HW libraries have the yaml in
-# `.github/.OwlBot.yaml`
-monorepo="false"
-if [[ -f "$(pwd)/.OwlBot.yaml" ]]; then
-  monorepo="true"
-fi
-
-if [[ "${monorepo}" == "true" ]]; then
+if [[ "${is_monorepo}" == "true" ]]; then
   mv owl-bot-staging/* temp
   rm -rd owl-bot-staging/
   mv temp owl-bot-staging
@@ -50,7 +41,7 @@ fi
 
 # apply repo templates
 echo "Rendering templates"
-python3 "${scripts_root}/owlbot/src/apply_repo_templates.py" "${configuration_yaml}" "${monorepo}"
+python3 "${scripts_root}/owlbot/src/apply_repo_templates.py" "${configuration_yaml}" "${is_monorepo}"
 
 # templates as well as retrieving files from owl-bot-staging
 echo "Retrieving files from owl-bot-staging directory..."
@@ -66,7 +57,7 @@ echo "...done"
 
 # write or restore pom.xml files
 echo "Generating missing pom.xml..."
-python3 "${scripts_root}/owlbot/src/fix-poms.py" "${versions_file}" "${monorepo}"
+python3 "${scripts_root}/owlbot/src/fix-poms.py" "${versions_file}" "${is_monorepo}"
 echo "...done"
 
 # write or restore clirr-ignored-differences.xml

diff --git a/library_generation/postprocess_library.sh b/library_generation/postprocess_library.sh
@@ -9,7 +9,7 @@
 # has the following requirements
 #   -  a .repo-metadata.json file must be present
 #   -  an owlbot.py file must be present
-#   -  an .OwlBot.yaml file must be present
+#   -  an .OwlBot-hermetic.yaml file must be present
 # 2 - preprocessed_sources_path: used to transfer the raw grpc, proto and gapic
 # libraries into the postprocessing_target via copy-code
 # 3 - versions_file: path to file containing versions to be applied to the poms
@@ -33,6 +33,7 @@ owlbot_cli_image_sha=$5
 synthtool_commitish=$6
 is_monorepo=$7
 configuration_yaml_path=$8
+owlbot_yaml_file_name=".OwlBot-hermetic.yaml"
 
 source "${scripts_root}"/utils/utilities.sh
 
@@ -44,7 +45,7 @@ for required_input in "${required_inputs[@]}"; do
   fi
 done
 
-for owlbot_file in ".repo-metadata.json" "owlbot.py" ".OwlBot.yaml"
+for owlbot_file in ".repo-metadata.json" "owlbot.py" "${owlbot_yaml_file_name}"
 do
   if [[ $(find "${postprocessing_target}" -name "${owlbot_file}" | wc -l) -eq 0 ]]; then
     echo "necessary file for postprocessing '${owlbot_file}' was not found in postprocessing_target"
@@ -59,10 +60,10 @@ if [[ -z "${owlbot_cli_source_folder}" ]]; then
 fi
 
 
-# we determine the location of the .OwlBot.yaml file by checking if the target
+# we determine the location of the .OwlBot-hermetic.yaml file by checking if the target
 # folder is a monorepo folder or not
 if [[ "${is_monorepo}" == "true" ]]; then
-  # the deep-remove-regex and deep-preserve-regex of the .OwlBot.yaml
+  # the deep-remove-regex and deep-preserve-regex of the .OwlBot-hermetic.yaml
   # files in the monorepo libraries assume that `copy-code` is run
   # from the root of the monorepo. However, we call `copy-code` from inside each
   # library, so a path like `/java-asset/google-.*/src` will not have
@@ -74,12 +75,12 @@ if [[ "${is_monorepo}" == "true" ]]; then
   # - "/google-.*/src"
 
   library_name=$(basename "${postprocessing_target}")
-  cat "${postprocessing_target}/.OwlBot.yaml" \
+  cat "${postprocessing_target}/${owlbot_yaml_file_name}" \
     | sed "s/- \"\/${library_name}/ - \"/" \
     > "${postprocessing_target}/.OwlBot.hermetic.yaml"
   owlbot_yaml_relative_path=".OwlBot.hermetic.yaml"
 else
-  owlbot_yaml_relative_path=".github/.OwlBot.yaml"
+  owlbot_yaml_relative_path=".github/${owlbot_yaml_file_name}"
 fi
 
 # Default values for running copy-code directly from host
@@ -150,5 +151,5 @@ popd # temp dir
 # run the postprocessor
 echo 'running owl-bot post-processor'
 pushd "${postprocessing_target}"
-bash "${scripts_root}/owlbot/bin/entrypoint.sh" "${scripts_root}" "${versions_file}" "${configuration_yaml_path}"
+bash "${scripts_root}/owlbot/bin/entrypoint.sh" "${scripts_root}" "${versions_file}" "${configuration_yaml_path}" "${is_monorepo}"
 popd # postprocessing_target
diff --git a/...est/resources/goldens/.OwlBot-golden.yaml → ...rces/goldens/.OwlBot-hermetic-golden.yaml b/...est/resources/goldens/.OwlBot-golden.yaml → ...rces/goldens/.OwlBot-hermetic-golden.yaml
diff --git a/...ry_generation/test/resources/integration/google-cloud-java/current_generation_config.yaml b/...ry_generation/test/resources/integration/google-cloud-java/current_generation_config.yaml
@@ -1,8 +1,8 @@
-gapic_generator_version: 2.37.0
+gapic_generator_version: 2.38.1
 protobuf_version: 25.2
 googleapis_commitish: 4ce0ff67a3d4509be641cbe47a35844ddc1268fc
 owlbot_cli_image: sha256:623647ee79ac605858d09e60c1382a716c125fb776f69301b72de1cd35d49409
-synthtool_commitish: 5e1fb2032fa44bc170677b38713023b4fec51a4e
+synthtool_commitish: 6612ab8f3afcd5e292aecd647f0fa68812c9f5b5
 template_excludes:
   - ".github/*"
   - ".kokoro/*"

diff --git a/library_generation/test/utilities_unit_tests.py b/library_generation/test/utilities_unit_tests.py
@@ -359,8 +359,8 @@ def test_generate_prerequisite_files_non_monorepo_success(self):
         # since this is a single library, we treat this as HW repository,
         # meaning that the owlbot yaml will be inside a .github folder
         file_comparator.compare_files(
-            f"{library_path}/.github/.OwlBot.yaml",
-            f"{library_path}/.OwlBot-golden.yaml",
+            f"{library_path}/.github/.OwlBot-hermetic.yaml",
+            f"{library_path}/.OwlBot-hermetic-golden.yaml",
         )
         file_comparator.compare_files(
             f"{library_path}/owlbot.py", f"{library_path}/owlbot-golden.py"
@@ -374,8 +374,8 @@ def test_generate_prerequisite_files_monorepo_success(self):
             f"{library_path}/.repo-metadata-monorepo-golden.json",
         )
         file_comparator.compare_files(
-            f"{library_path}/.OwlBot.yaml",
-            f"{library_path}/.OwlBot-golden.yaml",
+            f"{library_path}/.OwlBot-hermetic.yaml",
+            f"{library_path}/.OwlBot-hermetic-golden.yaml",
         )
         file_comparator.compare_files(
             f"{library_path}/owlbot.py", f"{library_path}/owlbot-golden.py"
@@ -432,7 +432,7 @@ def __setup_prerequisite_files(
         library_path = f"{resources_dir}/goldens"
         files = [
             f"{library_path}/.repo-metadata.json",
-            f"{library_path}/.OwlBot.yaml",
+            f"{library_path}/.OwlBot-hermetic.yaml",
             f"{library_path}/owlbot.py",
         ]
         cleanup(files)

diff --git a/library_generation/utils/utilities.py b/library_generation/utils/utilities.py
@@ -280,8 +280,8 @@ def generate_prerequisite_files(
         with open(f"{library_path}/{json_file}", "w") as fp:
             json.dump(repo_metadata, fp, indent=2)
 
-    # generate .OwlBot.yaml
-    owlbot_yaml_file = ".OwlBot.yaml"
+    # generate .OwlBot-hermetic.yaml
+    owlbot_yaml_file = ".OwlBot-hermetic.yaml"
     path_to_owlbot_yaml_file = (
         f"{library_path}/{owlbot_yaml_file}"
         if config.is_monorepo()

diff --git a/library_generation/utils/utilities.sh b/library_generation/utils/utilities.sh
@@ -276,7 +276,7 @@ get_proto_path_from_preprocessed_sources() {
 
 # for a pre-processed library stored in $preprocessed_sources_path, a folder
 # tree is built on $target_folder so it looks like a googleapis-gen folder and
-# is therefore consumable by an .OwlBot.yaml file
+# is therefore consumable by OwlBot CLI
 build_owlbot_cli_source_folder() {
   local postprocessing_target=$1
   local target_folder=$2
@@ -288,7 +288,7 @@ build_owlbot_cli_source_folder() {
   owlbot_staging_folder="${postprocessing_target}/owl-bot-staging"
   mkdir -p "${owlbot_staging_folder}"
 
-  # By default (thanks to generation templates), .OwlBot.yaml `deep-copy` section
+  # By default (thanks to generation templates), .OwlBot-hermetic.yaml `deep-copy` section
   # references a wildcard pattern matching a folder
   # ending with `-java` at the leaf of proto_path. We then use a generated-java
   # folder that will be picked up by copy-code