feat: enable hermetic library generation (#1667)

* restore to older version

* infer image tag from config yaml

* correct workflow name

* update config scripts and yamls

* remove old update_googleapis_committish workflow

* sync config structure with that of google-cloud-java

* remove quotes from config yamls

* fix typo in update_generation_config.yaml

* correct

* quote codeowners_team in generation config

* update generator version

* fix path to hermetic_library_generation

* remove escaped newlines

* fixes to hermetic_library_generation

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

---------

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>

.github/scripts/hermetic_library_generation.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+set -e
+# This script should be run at the root of the repository.
+# This script is used to, when a pull request changes the generation
+# configuration (generation_config.yaml by default):
+# 1. Find whether the last commit in this pull request contains changes to
+# the generation configuration and exit early if it doesn't have such a change
+# since the generation result would be the same.
+# 2. Compare generation configurations in the current branch (with which the
+# pull request associated) and target branch (into which the pull request is
+# merged);
+# 3. Generate changed libraries using library_generation image;
+# 4. Commit the changes to the pull request, if any.
+# 5. Edit the PR body with generated pull request description, if applicable.
+
+# The following commands need to be installed before running the script:
+# 1. git
+# 2. gh
+# 3. docker
+
+# The parameters of this script is:
+# 1. target_branch, the branch into which the pull request is merged.
+# 2. current_branch, the branch with which the pull request is associated.
+# 3. [optional] generation_config, the path to the generation configuration,
+# the default value is generation_config.yaml in the repository root.
+while [[ $# -gt 0 ]]; do
+key="$1"
+case "${key}" in
+ --target_branch)
+ target_branch="$2"
+ shift
+ ;;
+ --current_branch)
+ current_branch="$2"
+ shift
+ ;;
+ --generation_config)
+ generation_config="$2"
+ shift
+ ;;
+ *)
+ echo "Invalid option: [$1]"
+ exit 1
+ ;;
+esac
+shift
+done
+
+if [ -z "${target_branch}" ]; then
+ echo "missing required argument --target_branch"
+ exit 1
+fi
+
+if [ -z "${current_branch}" ]; then
+ echo "missing required argument --current_branch"
+ exit 1
+fi
+
+if [ -z "${generation_config}" ]; then
+ generation_config=generation_config.yaml
+ echo "Using default generation config: ${generation_config}"
+fi
+
+workspace_name="/workspace"
+baseline_generation_config="baseline_generation_config.yaml"
+message="chore: generate libraries at $(date)"
+
+git checkout "${target_branch}"
+git checkout "${current_branch}"
+# if the last commit doesn't contain changes to generation configuration,
+# do not generate again as the result will be the same.
+change_of_last_commit="$(git diff-tree --no-commit-id --name-only HEAD~1..HEAD -r)"
+if [[ ! ("${change_of_last_commit}" == *"${generation_config}"*) ]]; then
+ echo "The last commit doesn't contain any changes to the generation_config.yaml, skipping the whole generation process." || true
+ exit 0
+fi
+# copy generation configuration from target branch to current branch.
+git show "${target_branch}":"${generation_config}" > "${baseline_generation_config}"
+config_diff=$(diff "${generation_config}" "${baseline_generation_config}" || true)
+
+# parse image tag from the generation configuration.
+image_tag=$(grep "gapic_generator_version" "${generation_config}" | cut -d ':' -f 2 | xargs)
+
+# run hermetic code generation docker image.
+docker run \
+ --rm \
+ -u "$(id -u):$(id -g)" \
+ -v "$(pwd):${workspace_name}" \
+ gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \
+ --baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \
+ --current-generation-config-path="${workspace_name}/${generation_config}"
+
+
+# commit the change to the pull request.
+if [[ $(basename $(pwd)) == "google-cloud-java" ]]; then
+ git add java-* pom.xml gapic-libraries-bom/pom.xml versions.txt
+else
+ # The image leaves intermediate folders and files it works with. Here we remove them
+ rm -rdf output googleapis "${baseline_generation_config}"
+ git add --all -- ':!pr_description.txt'
+fi
+changed_files=$(git diff --cached --name-only)
+if [[ "${changed_files}" == "" ]]; then
+ echo "There is no generated code change with the generation config change ${config_diff}."
+ echo "Skip committing to the pull request."
+ exit 0
+fi
+
+echo "Configuration diff:"
+echo "${config_diff}"
+git commit -m "${message}"
+git push
+# set pr body if pr_description.txt is generated.
+if [[ -f "pr_description.txt" ]]; then
+ pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number")
+ gh pr edit "${pr_num}" --body "$(cat pr_description.txt)"
+fi

.github/scripts/update_generation_config.sh

@@ -0,0 +1,121 @@
+#!/bin/bash
+set -e
+# This script should be run at the root of the repository.
+# This script is used to update googleapis_commitish, gapic_generator_version,
+# and libraries_bom_version in generation configuration at the time of running
+# and create a pull request.
+
+# The following commands need to be installed before running the script:
+# 1. git
+# 2. gh
+# 3. jq
+
+# Utility functions
+# Get the latest released version of a Maven artifact.
+function get_latest_released_version() {
+ local group_id=$1
+ local artifact_id=$2
+ latest=$(curl -s "https://search.maven.org/solrsearch/select?q=g:${group_id}+AND+a:${artifact_id}&core=gav&rows=500&wt=json" | jq -r '.response.docs[] | select(.v | test("^[0-9]+(\\.[0-9]+)*$")) | .v' | sort -V | tail -n 1)
+ echo "${latest}"
+}
+
+# Update a key to a new value in the generation config.
+function update_config() {
+ local key_word=$1
+ local new_value=$2
+ local file=$3
+ echo "Update ${key_word} to ${new_value} in ${file}"
+ sed -i -e "s/^${key_word}.*$/${key_word}: ${new_value}/" "${file}"
+}
+
+# The parameters of this script is:
+# 1. base_branch, the base branch of the result pull request.
+# 2. repo, organization/repo-name, e.g., googleapis/google-cloud-java
+# 3. [optional] generation_config, the path to the generation configuration,
+# the default value is generation_config.yaml in the repository root.
+while [[ $# -gt 0 ]]; do
+key="$1"
+case "${key}" in
+ --base_branch)
+ base_branch="$2"
+ shift
+ ;;
+ --repo)
+ repo="$2"
+ shift
+ ;;
+ --generation_config)
+ generation_config="$2"
+ shift
+ ;;
+ *)
+ echo "Invalid option: [$1]"
+ exit 1
+ ;;
+esac
+shift
+done
+
+if [ -z "${base_branch}" ]; then
+ echo "missing required argument --base_branch"
+ exit 1
+fi
+
+if [ -z "${repo}" ]; then
+ echo "missing required argument --repo"
+ exit 1
+fi
+
+if [ -z "${generation_config}" ]; then
+ generation_config="generation_config.yaml"
+ echo "Use default generation config: ${generation_config}"
+fi
+
+current_branch="generate-libraries-${base_branch}"
+title="chore: Update generation configuration at $(date)"
+
+# try to find a open pull request associated with the branch
+pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number")
+# create a branch if there's no open pull request associated with the
+# branch; otherwise checkout the pull request.
+if [ -z "${pr_num}" ]; then
+ git checkout -b "${current_branch}"
+else
+ gh pr checkout "${pr_num}"
+fi
+
+mkdir tmp-googleapis
+# use partial clone because only commit history is needed.
+git clone --filter=blob:none https://github.com/googleapis/googleapis.git tmp-googleapis
+pushd tmp-googleapis
+git pull
+latest_commit=$(git rev-parse HEAD)
+popd
+rm -rf tmp-googleapis
+update_config "googleapis_commitish" "${latest_commit}" "${generation_config}"
+
+# update gapic-generator-java version to the latest
+latest_version=$(get_latest_released_version "com.google.api" "gapic-generator-java")
+update_config "gapic_generator_version" "${latest_version}" "${generation_config}"
+
+# update libraries-bom version to the latest
+latest_version=$(get_latest_released_version "com.google.cloud" "libraries-bom")
+update_config "libraries_bom_version" "${latest_version}" "${generation_config}"
+
+git add "${generation_config}"
+changed_files=$(git diff --cached --name-only)
+if [[ "${changed_files}" == "" ]]; then
+ echo "The latest generation config is not changed."
+ echo "Skip committing to the pull request."
+ exit 0
+fi
+git commit -m "${title}"
+if [ -z "${pr_num}" ]; then
+ git remote add remote_repo https://cloud-java-bot:"${GH_TOKEN}@github.com/${repo}.git"
+ git fetch -q --unshallow remote_repo
+ git push -f remote_repo "${current_branch}"
+ gh pr create --title "${title}" --head "${current_branch}" --body "${title}" --base "${base_branch}"
+else
+ git push
+ gh pr edit "${pr_num}" --title "${title}" --body "${title}"
+fi

.github/workflows/hermetic_library_generation.yaml

@@ -0,0 +1,40 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# GitHub action job to test core java library features on
+# downstream client libraries before they are released.
+name: Hermetic library generation upon generation config change through pull requests
+on:
+ pull_request:
+
+jobs:
+ library_generation:
+ # skip pull requests coming from a forked repository
+ if: github.event.pull_request.head.repo.full_name == github.repository
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}
+ - name: Generate changed libraries
+ shell: bash
+ run: |
+ set -x
+ [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com"
+ [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot"
+ bash .github/scripts/hermetic_library_generation.sh \
+ --target_branch ${{ github.base_ref }} \
+ --current_branch ${{ github.head_ref }}
+ env:
+ GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}

.github/workflows/update_generation_config.yaml

@@ -0,0 +1,42 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# GitHub action job to test core java library features on
+# downstream client libraries before they are released.
+name: Update generation configuration
+on:
+ schedule:
+ - cron: '0 2 * * *'
+ workflow_dispatch:
+
+jobs:
+ update-generation-config:
+ runs-on: ubuntu-22.04
+ env:
+ # the branch into which the pull request is merged
+ base_branch: main
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}
+ - name: Update params in generation config to latest
+ shell: bash
+ run: |
+ set -x
+ [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com"
+ [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot"
+ bash .github/scripts/update_generation_config.sh \
+ --base_branch "${base_branch}"\
+ --repo ${{ github.repository }}
+ env:
+ GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}

README.md

@@ -33,13 +33,13 @@ If you are using Maven, add this to your pom.xml file:
 If you are using Gradle without BOM, add this to your dependencies:
 
 ```Groovy
-implementation 'com.google.cloud:google-cloud-pubsublite:1.13.7'
+implementation 'com.google.cloud:google-cloud-pubsublite:1.13.8'
 ```
 
 If you are using SBT, add this to your dependencies:
 
 ```Scala
-libraryDependencies += "com.google.cloud" % "google-cloud-pubsublite" % "1.13.7"
+libraryDependencies += "com.google.cloud" % "google-cloud-pubsublite" % "1.13.8"
 ```
 <!-- {x-version-update-end} -->
 
@@ -483,7 +483,7 @@ Java is a registered trademark of Oracle and/or its affiliates.
 [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-pubsublite/java11.html
 [stability-image]: https://img.shields.io/badge/stability-stable-green
 [maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-pubsublite.svg
-[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-pubsublite/1.13.7
+[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-pubsublite/1.13.8
 [authentication]: https://github.com/googleapis/google-cloud-java#authentication
 [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes
 [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles

generation_config.yaml

@@ -0,0 +1,23 @@
+gapic_generator_version: 2.42.0
+googleapis_commitish: 0606cbcec6e416faa824c98a6d4eda4c22cfdb29
+libraries_bom_version: 26.38.0
+libraries:
+ - api_shortname: pubsublite
+ name_pretty: Cloud Pub/Sub Lite
+ product_documentation: https://cloud.google.com/pubsub/lite/docs
+ api_description: is designed to provide reliable, many-to-many, asynchronous messaging between applications. Publisher\napplications can send messages to a topic and other applications can\nsubscribe to that topic to receive the messages. By decoupling senders and\nreceivers, Google Cloud Pub/Sub allows developers to communicate between\nindependently written applications.\n\nCompared to Google Pub/Sub, Pub/Sub Lite provides partitioned zonal data\nstorage with predefined capacity. Both products present a similar API, but\nPub/Sub Lite has more usage caveats.\n\nSee the [Google Pub/Sub Lite docs](https://cloud.google.com/pubsub/quickstart-console#before-you-begin) for more details on how to activate\nPub/Sub Lite for your project, as well as guidance on how to choose between\nCloud Pub/Sub and Pub/Sub Lite.
+ client_documentation: https://cloud.google.com/java/docs/reference/google-cloud-pubsublite/latest/history
+ release_level: stable
+ transport: grpc
+ requires_billing: true
+ language: java
+ min_java_version: 8
+ repo: googleapis/java-pubsublite
+ repo_short: java-pubsublite
+ distribution_name: com.google.cloud:google-cloud-pubsublite
+ codeowner_team: '@googleapis/api-pubsublite'
+ library_type: GAPIC_COMBO
+ api_id: pubsublite.googleapis.com
+ recommended_package: com.google.cloud.pubsublite.cloudpubsub
+ GAPICs:
+ - proto_path: google/cloud/pubsublite/v1