feat: add config change (#2604)

In this PR:
- Add `config_change.py` to get:
- Changed libraries, which will be passed to `generate_repo.py` to
generate libraries selectedly.
- Qualified commits, which will be passed to
`generate_pr_description.py` to generate PR description.
- Refactor part of utility methods to `proto_path_utils.py`.
- Refactor `generation_config_comparator.py` to move data class to
`config_change.py`.
- Refactor `utilities.py` and `utilities.sh` to `utils/utilities.py` and
`utils/utilities.sh`.
- Add unit tests.

Follow up of #2587

For the goal of series of PRs, please refer to [improvement
proposal](https://docs.google.com/document/d/1JiCcG3X7lnxaJErKe0ES_JkyU7ECb40nf2Xez3gWvuo/edit?tab=t.g3vua2kd06gx#bookmark=id.72s3ukwwzevo).

library_generation/generate_composed_library.py

@@ -30,7 +30,7 @@
 import os
 from pathlib import Path
 from typing import List
-import library_generation.utilities as util
+import library_generation.utils.utilities as util
 from library_generation.model.generation_config import GenerationConfig
 from library_generation.model.gapic_config import GapicConfig
 from library_generation.model.gapic_inputs import GapicInputs

library_generation/generate_library.sh

@@ -74,7 +74,7 @@ done
 
 script_dir=$(dirname "$(readlink -f "$0")")
 # source utility functions
-source "${script_dir}"/utilities.sh
+source "${script_dir}"/utils/utilities.sh
 output_folder="$(get_output_folder)"
 
 if [ -z "${gapic_generator_version}" ]; then

library_generation/generate_pr_description.py

@@ -19,10 +19,8 @@
 import click
 from git import Commit, Repo
 from library_generation.model.generation_config import from_yaml
-from library_generation.utilities import find_versioned_proto_path
+from library_generation.utils.proto_path_utils import find_versioned_proto_path
 from library_generation.utils.commit_message_formatter import format_commit_message
-from library_generation.utilities import get_file_paths
-from library_generation.utils.commit_message_formatter import wrap_nested_commit
 from library_generation.utils.commit_message_formatter import wrap_override_commit
 
 
@@ -87,7 +85,7 @@ def generate_pr_descriptions(
     baseline_commit: str,
 ) -> str:
     config = from_yaml(generation_config_yaml)
-    paths = get_file_paths(config)
+    paths = config.get_proto_path_to_library_name()
     return __get_commit_messages(
         repo_url=repo_url,
         latest_commit=config.googleapis_commitish,

library_generation/generate_repo.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import library_generation.utilities as util
+import library_generation.utils.utilities as util
 import click
 import os
 from library_generation.generate_composed_library import generate_composed_library

library_generation/model/config_change.py

@@ -0,0 +1,158 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+import shutil
+from enum import Enum
+from typing import Optional
+from git import Commit, Repo
+from library_generation.model.generation_config import GenerationConfig
+from library_generation.model.library_config import LibraryConfig
+from library_generation.utils.utilities import sh_util
+from library_generation.utils.proto_path_utils import find_versioned_proto_path
+
+
+class ChangeType(Enum):
+    GOOGLEAPIS_COMMIT = 1
+    REPO_LEVEL_CHANGE = 2
+    LIBRARIES_ADDITION = 3
+    # As of Mar. 2024, we decide not to produce this type of change because we
+    # still need to manually remove the libray.
+    # LIBRARIES_REMOVAL = 4
+    LIBRARY_LEVEL_CHANGE = 5
+    GAPIC_ADDITION = 6
+    # As of Mar. 2024, we decide not to produce this type of change because we
+    # still need to manually remove the libray.
+    # GAPIC_REMOVAL = 7
+
+
+class HashLibrary:
+    """
+    Data class to group a LibraryConfig object and its hash value together.
+    """
+
+    def __init__(self, hash_value: int, library: LibraryConfig):
+        self.hash_value = hash_value
+        self.library = library
+
+
+class LibraryChange:
+    def __init__(self, changed_param: str, latest_value: str, library_name: str = ""):
+        self.changed_param = changed_param
+        self.latest_value = latest_value
+        self.library_name = library_name
+
+
+class QualifiedCommit:
+    def __init__(self, commit: Commit, libraries: set[str]):
+        self.commit = commit
+        self.libraries = libraries
+
+
+class ConfigChange:
+    ALL_LIBRARIES_CHANGED = None
+
+    def __init__(
+        self,
+        change_to_libraries: dict[ChangeType, list[LibraryChange]],
+        baseline_config: GenerationConfig,
+        latest_config: GenerationConfig,
+    ):
+        self.change_to_libraries = change_to_libraries
+        self.baseline_config = baseline_config
+        self.latest_config = latest_config
+
+    def get_changed_libraries(self) -> Optional[list[str]]:
+        """
+        Returns a unique, sorted list of library name of changed libraries.
+        None if there is a repository level change, which means all libraries
+        in the latest_config will be generated.
+        :return: library names of change libraries.
+        """
+        if ChangeType.REPO_LEVEL_CHANGE in self.change_to_libraries:
+            return ConfigChange.ALL_LIBRARIES_CHANGED
+        library_names = set()
+        for change_type, library_changes in self.change_to_libraries.items():
+            if change_type == ChangeType.GOOGLEAPIS_COMMIT:
+                library_names.update(self.__get_library_names_from_qualified_commits())
+            else:
+                library_names.update(
+                    [library_change.library_name for library_change in library_changes]
+                )
+        return sorted(list(library_names))
+
+    def get_qualified_commits(
+        self,
+        repo_url: str = "https://github.com/googleapis/googleapis.git",
+    ) -> list[QualifiedCommit]:
+        """
+        Returns qualified commits from configuration change.
+
+        A qualified commit is a commit that changes at least one file (excluding
+        BUILD.bazel) within a versioned proto path in the given proto_paths.
+        :param repo_url: the repository contains the commit history.
+        :return: QualifiedCommit objects.
+        """
+        tmp_dir = sh_util("get_output_folder")
+        shutil.rmtree(tmp_dir, ignore_errors=True)
+        os.mkdir(tmp_dir)
+        # we only need commit history, thus shadow clone is enough.
+        repo = Repo.clone_from(url=repo_url, to_path=tmp_dir, filter=["blob:none"])
+        commit = repo.commit(self.latest_config.googleapis_commitish)
+        proto_paths = self.latest_config.get_proto_path_to_library_name()
+        qualified_commits = []
+        while str(commit.hexsha) != self.baseline_config.googleapis_commitish:
+            qualified_commit = ConfigChange.__create_qualified_commit(
+                proto_paths=proto_paths, commit=commit
+            )
+            if qualified_commit is not None:
+                qualified_commits.append(qualified_commit)
+            commit_parents = commit.parents
+            if len(commit_parents) == 0:
+                break
+            commit = commit_parents[0]
+        shutil.rmtree(tmp_dir, ignore_errors=True)
+        return qualified_commits
+
+    def __get_library_names_from_qualified_commits(self) -> list[str]:
+        qualified_commits = self.get_qualified_commits()
+        library_names = []
+        for qualified_commit in qualified_commits:
+            library_names.extend(qualified_commit.libraries)
+        return library_names
+
+    @staticmethod
+    def __create_qualified_commit(
+        proto_paths: dict[str, str], commit: Commit
+    ) -> Optional[QualifiedCommit]:
+        """
+        Returns a qualified commit from the given Commit object; otherwise None.
+
+        :param proto_paths: a mapping from versioned proto_path to library_name
+        :param commit: a GitHub commit object.
+        :return: qualified commits.
+        """
+        libraries = set()
+        for file in commit.stats.files.keys():
+            if file.endswith("BUILD.bazel"):
+                continue
+            versioned_proto_path = find_versioned_proto_path(file)
+            if versioned_proto_path in proto_paths:
+                # Even though a commit usually only changes one
+                # library, we don't want to miss generating a
+                # library because the commit may change multiple
+                # libraries.
+                libraries.add(proto_paths[versioned_proto_path])
+        if len(libraries) == 0:
+            return None
+        return QualifiedCommit(commit=commit, libraries=libraries)

library_generation/model/generation_config.py

@@ -49,6 +49,18 @@ def __init__(
         # monorepos have more than one library defined in the config yaml
         self.is_monorepo = len(libraries) > 1
 
+    def get_proto_path_to_library_name(self) -> dict[str, str]:
+        """
+        Get versioned proto_path to library_name mapping from configuration.
+
+        :return: versioned proto_path to library_name mapping
+        """
+        paths = {}
+        for library in self.libraries:
+            for gapic_config in library.gapic_configs:
+                paths[gapic_config.proto_path] = library.get_library_name()
+        return paths
+
 
 def from_yaml(path_to_yaml: str) -> GenerationConfig:
     """

library_generation/postprocess_library.sh

@@ -34,7 +34,7 @@ synthtool_commitish=$6
 is_monorepo=$7
 configuration_yaml_path=$8
 
-source "${scripts_root}"/utilities.sh
+source "${scripts_root}"/utils/utilities.sh
 
 declare -a required_inputs=("postprocessing_target" "versions_file" "owlbot_cli_image_sha" "synthtool_commitish" "is_monorepo")
 for required_input in "${required_inputs[@]}"; do

library_generation/setup.py

@@ -12,7 +12,9 @@
     },
     package_data={
         "library_generation": [
-            "*.sh",
+            "generate_library.sh",
+            "postprocess_library.sh",
+            "utils/utilities.sh",
             "templates/*.j2",
             "gapic-generator-java-wrapper",
             "requirements.*",

library_generation/test/compare_poms.py

@@ -4,7 +4,7 @@
 The only comparison points are: element path (e.g. project/dependencies) and element text
 There is a special case for `dependency`, where the maven coordinates are prepared as well
 """
-from library_generation.utilities import eprint
+from library_generation.utils.utilities import eprint
 import xml.etree.ElementTree as et
 from collections import Counter
 import sys

library_generation/test/generate_library_unit_tests.sh

@@ -5,7 +5,7 @@ set -xeo pipefail
 # Unit tests against ../utilities.sh
 script_dir=$(dirname "$(readlink -f "$0")")
 source "${script_dir}"/test_utilities.sh
-source "${script_dir}"/../utilities.sh
+source "${script_dir}"/../utils/utilities.sh
 
 # Unit tests
 extract_folder_name_test() {

library_generation/test/integration_tests.py

@@ -28,7 +28,7 @@
 from library_generation.generate_repo import generate_from_yaml
 from library_generation.model.generation_config import from_yaml, GenerationConfig
 from library_generation.test.compare_poms import compare_xml
-from library_generation.utilities import (
+from library_generation.utils.utilities import (
     sh_util as shell_call,
     run_process_and_print_output,
 )