feat: bake gapic-generator-java into the hermetic build docker image

.cloudbuild/library_generation/library_generation.Dockerfile

@@ -12,11 +12,25 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# install gapic-generator-java in a separate layer so we don't overload the image
+# with the transferred source code and jars
+FROM gcr.io/cloud-devrel-public-resources/java21 AS ggj-build
+
+WORKDIR /sdk-platform-java
+COPY . .
+# {x-version-update-start:gapic-generator-java:current}
+ENV DOCKER_GAPIC_GENERATOR_VERSION="2.42.1-SNAPSHOT" 
+
+RUN mvn install -DskipTests -Dclirr.skip -Dcheckstyle.skip
+RUN ls "/root/.m2/repository/com/google/api/gapic-generator-java/" 
+RUN cp "/root/.m2/repository/com/google/api/gapic-generator-java/${DOCKER_GAPIC_GENERATOR_VERSION}/gapic-generator-java-${DOCKER_GAPIC_GENERATOR_VERSION}.jar" .
+
 # build from the root of this repo:
 FROM gcr.io/cloud-devrel-public-resources/python
 
 SHELL [ "/bin/bash", "-c" ]
 
+
 ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550
 ARG PROTOC_VERSION=25.3
 ARG GRPC_VERSION=1.65.1
@@ -47,7 +61,16 @@ RUN source /src/utils/utilities.sh \
 ENV DOCKER_GRPC_LOCATION="/grpc/protoc-gen-grpc-java-${GRPC_VERSION}-${OS_ARCHITECTURE}.exe"
 ENV DOCKER_GRPC_VERSION="${GRPC_VERSION}"
 
-# use python 3.11 (the base image has several python versions; here we define the default one)
+
+# we transfer gapic-generator-java from the previous stage.
+# here we redeclare this env var since they are not preserved between stages
+ENV DOCKER_GAPIC_GENERATOR_VERSION="2.42.1-SNAPSHOT" 
+# {x-version-update-end:gapic-generator-java:current}
+ENV DOCKER_GAPIC_GENERATOR_LOCATION="/gapic-generator-java/gapic-generator-java-${DOCKER_GAPIC_GENERATOR_VERSION}.jar"
+COPY --from=ggj-build "/root/.m2/repository/com/google/api/gapic-generator-java/${DOCKER_GAPIC_GENERATOR_VERSION}/gapic-generator-java-${DOCKER_GAPIC_GENERATOR_VERSION}.jar" "${DOCKER_GAPIC_GENERATOR_LOCATION}"
+RUN chmod 755 "${DOCKER_GAPIC_GENERATOR_LOCATION}"
+
+#  use python 3.11 (the base image has several python versions; here we define the default one)
 RUN rm $(which python3)
 RUN ln -s $(which python3.11) /usr/local/bin/python
 RUN ln -s $(which python3.11) /usr/local/bin/python3

library_generation/generate_library.sh

@@ -16,8 +16,6 @@ case $key in
     ;;
   --gapic_generator_version)
     gapic_generator_version="$2"
-    # export this variable so that it can be used in gapic-generator-java-wrapper.sh
-    export gapic_generator_version
     shift
     ;;
   --protoc_version)
@@ -77,9 +75,16 @@ script_dir=$(dirname "$(readlink -f "$0")")
 source "${script_dir}"/utils/utilities.sh
 output_folder="$(get_output_folder)"
 
-if [ -z "${gapic_generator_version}" ]; then
-  echo 'missing required argument --gapic_generator_version'
-  exit 1
+if [[ -z "${gapic_generator_version}" ]] ; then
+  if [[ -n "${DOCKER_GAPIC_GENERATOR_VERSION}" ]]; then
+    # we fall back to the docker env var so it's handled downstream in
+    # utilities.sh. This will make the jar embedded in the image to be
+    # transferred to output_folder
+    gapic_generator_version="${DOCKER_GAPIC_GENERATOR_VERSION}"
+  else
+    echo 'missing required argument --gapic_generator_version'
+    exit 1
+  fi
 fi
 
 if [ -z "${protoc_version}" ]; then
@@ -126,6 +131,9 @@ if [ -z "${os_architecture}" ]; then
   os_architecture=$(detect_os_architecture)
 fi
 
+# export this variable so that it can be used in gapic-generator-java-wrapper.sh
+export gapic_generator_version
+
 temp_destination_path="${output_folder}/temp_preprocessed"
 mkdir -p "${output_folder}/${destination_path}"
 if [ -d "${temp_destination_path}" ]; then

library_generation/test/generate_library_unit_tests.sh

@@ -212,6 +212,50 @@ download_tools_succeed_with_baked_grpc() {
   unset grpc_path
 }
 
+download_tools_succeed_with_baked_generator() {
+  # This test has the same structure as
+  # download_tools_succeed_with_baked_protoc, but meant for
+  # gapic-generator-java.
+  local test_dir=$(mktemp -d)
+  pushd "${test_dir}"
+  generator_folder=$(mktemp -d)
+  touch "${generator_folder}/gapic-generator-java.fakejar"
+  export DOCKER_GAPIC_GENERATOR_LOCATION="${generator_folder}/gapic-generator-java.fakejar"
+  export DOCKER_GAPIC_GENERATOR_VERSION="99.99"
+  export output_folder=$(get_output_folder)
+  mkdir "${output_folder}"
+
+  local test_protoc_version="1.64.0"
+  local test_grpc_version="1.64.0"
+  # we expect download_tools to decide to use DOCKER_GAPIC_GENERATOR_LOCATION because
+  # the protoc version we want to download is the same as DOCKER_GRPC_VERSION
+  log=$(download_tools "99.99" "${test_protoc_version}" "${test_grpc_version}" "linux-x86_64" 2>&1 1>/dev/null)
+
+  # the assertion functions are designed to be called only once per function.
+  # Here we do two manual tests that are then coverged into a single
+  # pseudo-boolean
+  has_expected_log="false"
+  has_expected_file="false"
+  is_output_correct="false"
+  if grep -q "Using gapic-generator-java version baked into the container: 99.99" <<< "${log}"; then
+    has_expected_log="true"
+  fi
+  if [[ -f "${output_folder}/gapic-generator-java.fakejar" ]]; then
+    has_expected_file="true"
+  fi
+  if [[ "${has_expected_log}" == "true" ]] && [[ "${has_expected_file}" == "true" ]]; then
+    is_output_correct="true"
+  fi
+
+  assertEquals "true" "${is_output_correct}"
+
+  rm -rdf "${output_folder}"
+  unset DOCKER_GRPC_LOCATION
+  unset DOCKER_GRPC_VERSION
+  unset output_folder
+  unset grpc_path
+}
+
 download_grpc_plugin_succeed_with_valid_version_linux_test() {
   download_grpc_plugin "1.55.1" "linux-x86_64"
   assertFileOrDirectoryExists "protoc-gen-grpc-java-1.55.1-linux-x86_64.exe"
@@ -350,6 +394,7 @@ test_list=(
   download_protoc_failed_with_invalid_arch_test
   download_tools_succeed_with_baked_protoc
   download_tools_succeed_with_baked_grpc
+  download_tools_succeed_with_baked_generator
   download_grpc_plugin_succeed_with_valid_version_linux_test
   download_grpc_plugin_succeed_with_valid_version_macos_test
   download_grpc_plugin_failed_with_invalid_version_linux_test

library_generation/test/test_utilities.sh

@@ -27,7 +27,7 @@ __test_failed() {
 }
 
 
-############# Functions used in test execution #############
+############# Functions used in test execution. They can only be called once per test #############
 
 assertEquals() {
   local expected=$1

library_generation/utils/utilities.sh

@@ -158,7 +158,6 @@ download_tools() {
   local grpc_version=$3
   local os_architecture=$4
   pushd "${output_folder}"
-  download_generator_artifact "${gapic_generator_version}" "gapic-generator-java-${gapic_generator_version}.jar"
 
   # the variable protoc_path is used in generate_library.sh. It is explicitly
   # exported to make clear that it is used outside this utilities file.
@@ -179,6 +178,19 @@ download_tools() {
     export grpc_path=$(download_grpc_plugin "${grpc_version}" "${os_architecture}")
   fi
 
+  # similar case with gapic-generator-java
+  if [[ "${DOCKER_GAPIC_GENERATOR_VERSION}" == "${gapic_generator_version}" ]]; then
+    # if the specified gapic_generator_version matches the one baked in the docker
+    # container, we copy the generator jar into the output folder so it can be
+    # picked up by gapic-generator-java-wrapper
+    >&2 echo "Using gapic-generator-java version baked into the container: ${DOCKER_GAPIC_GENERATOR_VERSION}"
+    cp "${DOCKER_GAPIC_GENERATOR_LOCATION}" "${output_folder}"
+  else
+    download_generator_artifact \
+      "${gapic_generator_version}" \
+      "gapic-generator-java-${gapic_generator_version}.jar"
+  fi
+
   popd
 }