chore: [hermetic-build] simplify usage of synthtool (#2772)

This PR eliminates the need for:
 * `library_generation/owlbot/src/apply_repo_templates.py`
 * Passing the config yaml path to the postprocessing scripts

 ### How is it achieved?
 We set the environment variable `SYNTHTOOL_TEMPLATES` to point to a
 temp dir where we copy the contents of `owlbot/templates` plus a
 modification of the template using `sed` (instead of
 `synthtool.common_templates`). In this way, we avoid the usage of
 `apply_repo_templates.py`.
 generation_config.template_excludes.

 #### Why don't we use common_templates to pass libraries_bom_version?
 Because `owlbot.py` is a legacy script that was written when
 `java.common_templates` didn't take `libraries_bom_version` as a
 parameter. If we don't like this approach of having two template
 replacement workflows we could

 1. Modify all owlbot.py files
 2. Pass libraries_bom_version as an environment variable (not bad)

 #### What about template_excludes in the config yaml?
 We still need it to generate new libraries. This is done by rendering a
 new `owlbot.py` via templates.

.cloudbuild/library_generation/library_generation.Dockerfile

@@ -15,7 +15,7 @@
 # build from the root of this repo:
 FROM gcr.io/cloud-devrel-public-resources/python
 
-ARG SYNTHTOOL_COMMITTISH=63cc541da2c45fcfca2136c43e638da1fbae174d
+ARG SYNTHTOOL_COMMITTISH=a2c9b4a5da2d7f583c8a1869fd2843c206145834
 ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550
 ENV HOME=/home
 

library_generation/cli/entry_point.py

@@ -117,7 +117,6 @@ def generate(
         # baseline_generation_config is not specified.
         # Do not generate pull request description.
         generate_from_yaml(
-            config_path=current_generation_config_path,
             config=from_yaml(current_generation_config_path),
             repository_path=repository_path,
         )
@@ -131,7 +130,6 @@ def generate(
         current_config=from_yaml(current_generation_config_path),
     )
     generate_from_yaml(
-        config_path=current_generation_config_path,
         config=config_change.current_config,
         repository_path=repository_path,
         target_library_names=config_change.get_changed_libraries(),

library_generation/generate_composed_library.py

@@ -41,7 +41,6 @@
 
 
 def generate_composed_library(
-    config_path: str,
     config: GenerationConfig,
     library_path: str,
     library: LibraryConfig,
@@ -51,7 +50,6 @@ def generate_composed_library(
     """
     Generate libraries composed of more than one service or service version
 
-    :param config_path: Path to generation configuration.
     :param config: a GenerationConfig object representing a parsed configuration
     yaml
     :param library_path: the path to which the generated file goes
@@ -113,7 +111,7 @@ def generate_composed_library(
             versions_file,
             owlbot_cli_source_folder,
             str(config.is_monorepo()).lower(),
-            config_path,
+            config.libraries_bom_version,
         ],
         "Library postprocessing",
     )

library_generation/generate_repo.py

@@ -20,7 +20,6 @@
 
 
 def generate_from_yaml(
-    config_path: str,
     config: GenerationConfig,
     repository_path: str,
     target_library_names: list[str] = None,
@@ -29,7 +28,6 @@ def generate_from_yaml(
     Based on the generation config, generates libraries via
     generate_composed_library.py
 
-    :param config_path: Path to generation configuration.
     :param config: a GenerationConfig object.
     :param repository_path: The repository path to which the generated files
     will be sent.
@@ -49,7 +47,6 @@ def generate_from_yaml(
     for library_path, library in repo_config.libraries.items():
         print(f"generating library {library.get_library_name()}")
         generate_composed_library(
-            config_path=config_path,
             config=config,
             library_path=library_path,
             library=library,

library_generation/model/generation_config.py

@@ -40,7 +40,7 @@ def __init__(
     ):
         self.gapic_generator_version = gapic_generator_version
         self.googleapis_commitish = googleapis_commitish
-        self.libraris_bom_version = libraries_bom_version
+        self.libraries_bom_version = libraries_bom_version
         self.template_excludes = template_excludes
         self.libraries = libraries
         self.grpc_version = grpc_version

library_generation/owlbot/bin/entrypoint.sh

@@ -16,18 +16,21 @@
 # This is the entrypoint script for java owlbot. This is not intended to be
 # called directly but rather be called from postproces_library.sh
 # For reference, the positional arguments are
-# 1: scripts_root: location of postprocess_library.sh
+# 1: scripts_root: location of the root of the library_generation scripts. When
+# in a Docker container, this value should be /src/
 # 2: versions_file: points to a versions.txt containing versions to be applied
 # both to README and pom.xml files
+# 3: is_monorepo: whether we are postprocessing a monorepo
+# 4: libraries_bom_version: used to render the readme
 
 # The scripts assumes the CWD is the folder where postprocessing is going to be
 # applied
 
 set -ex
 scripts_root=$1
 versions_file=$2
-configuration_yaml=$3
-is_monorepo=$4
+is_monorepo=$3
+libraries_bom_version=$4
 
 
 if [[ "${is_monorepo}" == "true" ]]; then
@@ -36,22 +39,19 @@ if [[ "${is_monorepo}" == "true" ]]; then
   mv temp owl-bot-staging
 fi
 
-
-# Runs template and etc in current working directory
-
-# apply repo templates
-echo "Rendering templates"
-python3 "${scripts_root}/owlbot/src/apply_repo_templates.py" "${configuration_yaml}" "${is_monorepo}"
-
 # templates as well as retrieving files from owl-bot-staging
 echo "Retrieving files from owl-bot-staging directory..."
 if [ -f "owlbot.py" ]
 then
-  # we use an empty synthtool folder to prevent cached templates from being used
-  export SYNTHTOOL_TEMPLATES=$(mktemp -d)
+  # we copy the templates to a temp folder because we need to do a special
+  # modification regarding libraries_bom_version that can't be handled by the
+  # synthtool library considering the way owlbot.py files are written
+  export SYNTHTOOL_TEMPLATES="${scripts_root}/owlbot/templates"
+  export SYNTHTOOL_LIBRARIES_BOM_VERSION="${libraries_bom_version}"
   # defaults to run owlbot.py
   python3 owlbot.py
-  export SYNTHTOOL_TEMPLATES=""
+  unset SYNTHTOOL_TEMPLATES
+  unset SYNTHTOOL_LIBRARIES_BOM_VERSION
 fi
 echo "...done"
 

library_generation/postprocess_library.sh

@@ -17,8 +17,8 @@
 # googleapis-gen. It will be used instead of preprocessed_sources_path if
 # 5 - is_monorepo: whether this library is a monorepo, which implies slightly
 # different logic
-# 6 - configuration_yaml_path: path to the configuration yaml containing library
-# generation information for this library
+# 6 - libraries_bom_version: used by our implementation of owlbot to render the
+# readme
 set -exo pipefail
 scripts_root=$(dirname "$(readlink -f "$0")")
 
@@ -27,7 +27,7 @@ preprocessed_sources_path=$2
 versions_file=$3
 owlbot_cli_source_folder=$4
 is_monorepo=$5
-configuration_yaml_path=$6
+libraries_bom_version=$6
 owlbot_yaml_file_name=".OwlBot-hermetic.yaml"
 
 source "${scripts_root}"/utils/utilities.sh
@@ -98,5 +98,10 @@ fi
 # run the postprocessor
 echo 'running owl-bot post-processor'
 pushd "${postprocessing_target}"
-bash "${scripts_root}/owlbot/bin/entrypoint.sh" "${scripts_root}" "${versions_file}" "${configuration_yaml_path}" "${is_monorepo}"
+bash "${scripts_root}/owlbot/bin/entrypoint.sh" \
+  "${scripts_root}" \
+  "${versions_file}" \
+  "${is_monorepo}" \
+  "${libraries_bom_version}"
+
 popd # postprocessing_target

library_generation/test/integration_tests.py

@@ -183,8 +183,10 @@ def test_entry_point_running_in_container(self):
 
     @classmethod
     def __build_image(cls, docker_file: str, cwd: str):
+        # we build the docker image without removing intermediate containers so
+        # we can re-test more quickly
         subprocess.check_call(
-            ["docker", "build", "--rm", "-f", docker_file, "-t", image_tag, "."],
+            ["docker", "build", "-f", docker_file, "-t", image_tag, "."],
             cwd=cwd,
         )