feat: bake gapic-generator-java into the hermetic build docker image

.cloudbuild/library_generation/library_generation.Dockerfile

@@ -12,11 +12,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# install gapic-generator-java in a separate layer so we don't overload the image
+# with the transferred source code and jars
+FROM gcr.io/cloud-devrel-public-resources/java21 AS ggj-build
+
+WORKDIR /sdk-platform-java
+COPY . .
+# {x-version-update-start:gapic-generator-java:current}
+ENV DOCKER_GAPIC_GENERATOR_VERSION="2.42.1-SNAPSHOT" 
+
+RUN mvn install -DskipTests -Dclirr.skip -Dcheckstyle.skip
+RUN ls "/root/.m2/repository/com/google/api/gapic-generator-java/" 
+RUN cp "/root/.m2/repository/com/google/api/gapic-generator-java/${DOCKER_GAPIC_GENERATOR_VERSION}/gapic-generator-java-${DOCKER_GAPIC_GENERATOR_VERSION}.jar" .
+
 # build from the root of this repo:
 FROM gcr.io/cloud-devrel-public-resources/python
 
 SHELL [ "/bin/bash", "-c" ]
 
+
 ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550
 ARG PROTOC_VERSION=25.3
 ARG GRPC_VERSION=1.65.1
@@ -47,7 +61,16 @@ RUN source /src/utils/utilities.sh \
 ENV DOCKER_GRPC_LOCATION="/grpc/protoc-gen-grpc-java-${GRPC_VERSION}-${OS_ARCHITECTURE}.exe"
 ENV DOCKER_GRPC_VERSION="${GRPC_VERSION}"
 
-# use python 3.11 (the base image has several python versions; here we define the default one)
+
+# we transfer gapic-generator-java from the previous stage.
+# here we redeclare this env var since they are not preserved between stages
+ENV DOCKER_GAPIC_GENERATOR_VERSION="2.42.1-SNAPSHOT" 
+# {x-version-update-end:gapic-generator-java:current}
+ENV DOCKER_GAPIC_GENERATOR_LOCATION="/gapic-generator-java/gapic-generator-java-${DOCKER_GAPIC_GENERATOR_VERSION}.jar"
+COPY --from=ggj-build "/root/.m2/repository/com/google/api/gapic-generator-java/${DOCKER_GAPIC_GENERATOR_VERSION}/gapic-generator-java-${DOCKER_GAPIC_GENERATOR_VERSION}.jar" "${DOCKER_GAPIC_GENERATOR_LOCATION}"
+RUN chmod 755 "${DOCKER_GAPIC_GENERATOR_LOCATION}"
+
+# use python 3.11 (the base image has several python versions; here we define the default one)
 RUN rm $(which python3)
 RUN ln -s $(which python3.11) /usr/local/bin/python
 RUN ln -s $(which python3.11) /usr/local/bin/python3

.github/scripts/hermetic_library_generation.sh

@@ -90,9 +90,6 @@ fi
 git show "${target_branch}":"${generation_config}" > "${baseline_generation_config}"
 config_diff=$(diff "${generation_config}" "${baseline_generation_config}" || true)
 
-generator_version=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout -pl gapic-generator-java)
-echo "Local generator version: ${generator_version}"
-
 # install generator locally since we're using a SNAPSHOT version.
 mvn -V -B -ntp clean install -DskipTests
 

library_generation/generate_library.sh

@@ -16,8 +16,6 @@ case $key in
  ;;
  --gapic_generator_version)
  gapic_generator_version="$2"
- # export this variable so that it can be used in gapic-generator-java-wrapper.sh
- export gapic_generator_version
  shift
  ;;
  --protoc_version)
@@ -77,7 +75,7 @@ script_dir=$(dirname "$(readlink -f "$0")")
 source "${script_dir}"/utils/utilities.sh
 output_folder="$(get_output_folder)"
 
-if [ -z "${gapic_generator_version}" ]; then
+if [[ -z "${gapic_generator_version}" ]] ; then
  echo 'missing required argument --gapic_generator_version'
  exit 1
 fi
@@ -126,6 +124,9 @@ if [ -z "${os_architecture}" ]; then
  os_architecture=$(detect_os_architecture)
 fi
 
+# export this variable so that it can be used in gapic-generator-java-wrapper.sh
+export gapic_generator_version
+
 temp_destination_path="${output_folder}/temp_preprocessed"
 mkdir -p "${output_folder}/${destination_path}"
 if [ -d "${temp_destination_path}" ]; then

library_generation/model/generation_config.py

@@ -25,7 +25,7 @@
 COMMON_PROTOS_LIBRARY_NAME = "common-protos"
 GAPIC_GENERATOR_VERSION = "gapic_generator_version"
 LIBRARIES_BOM_VERSION = "libraries_bom_version"
-GENERATOR_VERSION_ENV_KEY = "GENERATOR_VERSION"
+GENERATOR_VERSION_ENV_KEY = "DOCKER_GAPIC_GENERATOR_VERSION"
 
 
 class GenerationConfig:
@@ -90,9 +90,11 @@ def __set_generator_version(gapic_generator_version: Optional[str]) -> str:
  gapic_generator_version = os.getenv(GENERATOR_VERSION_ENV_KEY)
  if not gapic_generator_version:
  raise ValueError(
- f"Environment variable {GENERATOR_VERSION_ENV_KEY}"
- f" is not set when the generator version is not"
- f" specified in the generation config."
+ f"gapic_generator_version was not specified in the "
+ f"configuration yaml and the fall-back env var "
+ f"{GENERATOR_VERSION_ENV_KEY} was not found. At "
+ f"least one of them must be set to proceed with the "
+ f"generation."
  )
  return gapic_generator_version
 

library_generation/test/generate_library_unit_tests.sh

@@ -212,6 +212,50 @@ download_tools_succeed_with_baked_grpc() {
  unset grpc_path
 }
 
+download_tools_succeed_with_baked_generator() {
+ # This test has the same structure as
+ # download_tools_succeed_with_baked_protoc, but meant for
+ # gapic-generator-java.
+ local test_dir=$(mktemp -d)
+ pushd "${test_dir}"
+ generator_folder=$(mktemp -d)
+ touch "${generator_folder}/gapic-generator-java.fakejar"
+ export DOCKER_GAPIC_GENERATOR_LOCATION="${generator_folder}/gapic-generator-java.fakejar"
+ export DOCKER_GAPIC_GENERATOR_VERSION="99.99"
+ export output_folder=$(get_output_folder)
+ mkdir "${output_folder}"
+
+ local test_protoc_version="1.64.0"
+ local test_grpc_version="1.64.0"
+ # we expect download_tools to decide to use DOCKER_GAPIC_GENERATOR_LOCATION because
+ # the protoc version we want to download is the same as DOCKER_GRPC_VERSION
+ log=$(download_tools "99.99" "${test_protoc_version}" "${test_grpc_version}" "linux-x86_64" 2>&1 1>/dev/null)
+
+ # the assertion functions are designed to be called only once per function.
+ # Here we do two manual tests that are then coverged into a single
+ # pseudo-boolean
+ has_expected_log="false"
+ has_expected_file="false"
+ is_output_correct="false"
+ if grep -q "Using gapic-generator-java version baked into the container: 99.99" <<< "${log}"; then
+ has_expected_log="true"
+ fi
+ if [[ -f "${output_folder}/gapic-generator-java.fakejar" ]]; then
+ has_expected_file="true"
+ fi
+ if [[ "${has_expected_log}" == "true" ]] && [[ "${has_expected_file}" == "true" ]]; then
+ is_output_correct="true"
+ fi
+
+ assertEquals "true" "${is_output_correct}"
+
+ rm -rdf "${output_folder}"
+ unset DOCKER_GRPC_LOCATION
+ unset DOCKER_GRPC_VERSION
+ unset output_folder
+ unset grpc_path
+}
+
 download_grpc_plugin_succeed_with_valid_version_linux_test() {
  download_grpc_plugin "1.55.1" "linux-x86_64"
  assertFileOrDirectoryExists "protoc-gen-grpc-java-1.55.1-linux-x86_64.exe"
@@ -350,6 +394,7 @@ test_list=(
  download_protoc_failed_with_invalid_arch_test
  download_tools_succeed_with_baked_protoc
  download_tools_succeed_with_baked_grpc
+ download_tools_succeed_with_baked_generator
  download_grpc_plugin_succeed_with_valid_version_linux_test
  download_grpc_plugin_succeed_with_valid_version_macos_test
  download_grpc_plugin_failed_with_invalid_version_linux_test

library_generation/test/model/generation_config_unit_tests.py

@@ -56,20 +56,20 @@ def test_generation_config_default_value(self):
  def test_generation_config_with_generator_version_env_raise_exception(self):
  self.assertRaisesRegex(
  ValueError,
- "Environment variable GENERATOR_VERSION is not set",
+ "the fall-back env var DOCKER_GAPIC_GENERATOR_VERSION was not found",
  GenerationConfig,
  googleapis_commitish="",
  libraries=[],
  )
 
  def test_generation_config_set_generator_version_from_env(self):
- os.environ["GENERATOR_VERSION"] = "1.2.3"
+ os.environ["DOCKER_GAPIC_GENERATOR_VERSION"] = "1.2.3"
  config = GenerationConfig(
  googleapis_commitish="",
  libraries=[],
  )
  self.assertEqual("1.2.3", config.gapic_generator_version)
- os.environ.pop("GENERATOR_VERSION")
+ os.environ.pop("DOCKER_GAPIC_GENERATOR_VERSION")
 
  def test_from_yaml_succeeds(self):
  config = from_yaml(f"{test_config_dir}/generation_config.yaml")

library_generation/test/test_utilities.sh

@@ -27,7 +27,7 @@ __test_failed() {
 }
 
 
-############# Functions used in test execution #############
+############# Functions used in test execution. They can only be called once per test #############
 
 assertEquals() {
  local expected=$1

library_generation/utils/utilities.sh

@@ -158,7 +158,6 @@ download_tools() {
  local grpc_version=$3
  local os_architecture=$4
  pushd "${output_folder}"
- download_generator_artifact "${gapic_generator_version}" "gapic-generator-java-${gapic_generator_version}.jar"
 
  # the variable protoc_path is used in generate_library.sh. It is explicitly
  # exported to make clear that it is used outside this utilities file.
@@ -179,6 +178,19 @@ download_tools() {
  export grpc_path=$(download_grpc_plugin "${grpc_version}" "${os_architecture}")
  fi
 
+ # similar case with gapic-generator-java
+ if [[ "${DOCKER_GAPIC_GENERATOR_VERSION}" == "${gapic_generator_version}" ]]; then
+ # if the specified gapic_generator_version matches the one baked in the docker
+ # container, we copy the generator jar into the output folder so it can be
+ # picked up by gapic-generator-java-wrapper
+ >&2 echo "Using gapic-generator-java version baked into the container: ${DOCKER_GAPIC_GENERATOR_VERSION}"
+ cp "${DOCKER_GAPIC_GENERATOR_LOCATION}" "${output_folder}"
+ else
+ download_generator_artifact \
+ "${gapic_generator_version}" \
+ "gapic-generator-java-${gapic_generator_version}.jar"
+ fi
+
  popd
 }