chore: track customization in templates

requirements.txt

@@ -304,4 +304,4 @@ watchdog==2.1.9 \
 setuptools==65.5.1 \
     --hash=sha256:d0b9a8433464d5800cbe05094acf5c6d52a91bfac9b52bcfc4d41382be5d5d31 \
     --hash=sha256:e197a19aa8ec9722928f2206f8de752def0e4c9fc6953527360d1c36d94ddb2f
-    # via -r requirements.in
+    # via -r requirements.in

synthtool/gcp/common.py

@@ -21,8 +21,8 @@
 from pathlib import Path
 from typing import Dict, List, Optional
 import jinja2
+import yaml
 from datetime import date
-
 from synthtool import shell, _tracked_paths
 from synthtool.gcp import partials
 from synthtool.languages import node, node_mono_repo
@@ -49,10 +49,33 @@ def __init__(self, template_path: Optional[Path] = None):
         self._templates = templates.Templates(self._template_root)
         self.excludes = []  # type: List[str]
 
-    def _generic_library(self, directory: str, relative_dir=None, **kwargs) -> Path:
-        # load common repo meta information (metadata that's not language specific).
+    def _generic_library(
+        self,
+        directory: str,
+        relative_dir=None,
+        partial_files: List[str] = None,
+        **kwargs,
+    ) -> Path:
+        # load default values that used to render templates.
+        defaults_path = self._template_root / directory / "defaults"
+        if defaults_path.exists():
+            kwargs["metadata"]["defaults"] = {}
+            for default_file in defaults_path.glob("*-default.yaml"):
+                self.excludes.append(
+                    "defaults/{}".format(os.path.basename(default_file))
+                )
+                with open(default_file) as f:
+                    kwargs["metadata"]["defaults"].update(
+                        yaml.load(f, Loader=yaml.SafeLoader)
+                    )
+        # load common repo meta information (metadata that's not language
+        # specific).
         if "metadata" in kwargs:
-            self._load_generic_metadata(kwargs["metadata"], relative_dir=relative_dir)
+            self._load_generic_metadata(
+                kwargs["metadata"],
+                relative_dir=relative_dir,
+                partial_files=partial_files,
+            )
             # if no samples were found, don't attempt to render a
             # samples/README.md.
             if "samples" not in kwargs["metadata"] or not kwargs["metadata"]["samples"]:
@@ -321,11 +344,13 @@ def py_library(self, **kwargs) -> Path:
                     f.write(content)
         return ret
 
-    def java_library(self, **kwargs) -> Path:
+    def java_library(self, partial_files: List[str] = None, **kwargs) -> Path:
         # kwargs["metadata"] is required to load values from .repo-metadata.json
         if "metadata" not in kwargs:
             kwargs["metadata"] = {}
-        return self._generic_library("java_library", **kwargs)
+        return self._generic_library(
+            "java_library", partial_files=partial_files, **kwargs
+        )
 
     def node_library(self, **kwargs) -> Path:
         # TODO: once we've migrated all Node.js repos to either having
@@ -397,11 +422,13 @@ def render(self, template_name: str, **kwargs) -> Path:
         _tracked_paths.add(template)
         return template
 
-    def _load_generic_metadata(self, metadata: Dict, relative_dir=None):
+    def _load_generic_metadata(
+        self, metadata: Dict, relative_dir=None, partial_files: List[str] = None
+    ):
         """
         loads additional meta information from .repo-metadata.json.
         """
-        metadata["partials"] = partials.load_partials()
+        metadata["partials"] = partials.load_partials(partial_files)
 
         # Loads repo metadata information from the default location if it
         # hasn't already been set. Some callers may have already loaded repo

synthtool/gcp/partials.py

@@ -18,12 +18,15 @@
 from typing import Dict, List
 
 # these are the default locations to look up
-_DEFAULT_PARTIAL_FILES = [".readme-partials.yml", ".readme-partials.yaml"]
+_DEFAULT_PARTIAL_FILES = [
+    ".readme-partials.yml",
+    ".readme-partials.yaml",
+]
 
 
-def load_partials(files: List[str] = _DEFAULT_PARTIAL_FILES) -> Dict:
+def load_partials(files: List[str] = None) -> Dict:
     """
-    hand-crafted artisinal markdown can be provided in a .readme-partials.yml.
+    hand-crafted artisanal markdown can be provided in a .readme-partials.yml.
     The following fields are currently supported:
 
     body: custom body to include in the usage section of the document.
@@ -34,13 +37,15 @@ def load_partials(files: List[str] = _DEFAULT_PARTIAL_FILES) -> Dict:
     deprecation_warning: a warning to indicate that the library has been
         deprecated and a pointer to an alternate option
     """
+    if files is None:
+        files = _DEFAULT_PARTIAL_FILES
+    else:
+        files.extend(_DEFAULT_PARTIAL_FILES)
+    res = {}
     cwd_path = Path(os.getcwd())
-    partials_file = None
     for file in files:
         if os.path.exists(cwd_path / file):
             partials_file = cwd_path / file
-            break
-    if not partials_file:
-        return {}
-    with open(partials_file) as f:
-        return yaml.load(f, Loader=yaml.SafeLoader)
+            with open(partials_file) as f:
+                res.update(yaml.load(f, Loader=yaml.SafeLoader))
+    return res

synthtool/gcp/templates/java_library/.kokoro/nightly/integration.cfg

@@ -1,37 +1,31 @@
 # Format: //devtools/kokoro/config/proto/build.proto
 
-# Configure the docker image for kokoro-trampoline.
+{%- if 'defaults' in metadata and 'kokoro_nightly_env_vars' in metadata['defaults'] -%}
+{%- for default_key, default_value in metadata['defaults']['kokoro_nightly_env_vars'].items() -%}
+{%- if 'partials' in metadata
+  and ('kokoro_nightly_env_vars' not in metadata['partials']
+    or ('kokoro_nightly_env_vars' in metadata['partials']
+      and default_key not in metadata['partials']['kokoro_nightly_env_vars'])) -%}
+{%- if default_key == "ENABLE_FLAKYBOT" %}
 env_vars: {
-  key: "TRAMPOLINE_IMAGE"
-  value: "gcr.io/cloud-devrel-kokoro-resources/java8"
-}
-
-env_vars: {
-  key: "JOB_TYPE"
-  value: "integration"
-}
-# TODO: remove this after we've migrated all tests and scripts
-env_vars: {
-  key: "GCLOUD_PROJECT"
-  value: "java-docs-samples-testing"
-}
-
-env_vars: {
-  key: "GOOGLE_CLOUD_PROJECT"
-  value: "java-docs-samples-testing"
-}
-
-env_vars: {
-  key: "ENABLE_FLAKYBOT"
+  key: "{{ default_key }}"
   value: {% if migrated_split_repo %}"false"{% else %}"true"{% endif %}
 }
-
+{% else %}
 env_vars: {
-  key: "GOOGLE_APPLICATION_CREDENTIALS"
-  value: "secret_manager/java-it-service-account"
+  key: "{{ default_key }}"
+  value: "{{ default_value }}"
 }
+{% endif %}
+{%- endif -%}
+{%- endfor -%}
+{%- endif -%}
 
+{%- if 'partials' in metadata and 'kokoro_nightly_env_vars' in metadata['partials'] -%}
+{% for partial_key, partial_value in metadata['partials']['kokoro_nightly_env_vars'].items() %}
 env_vars: {
-  key: "SECRET_MANAGER_KEYS"
-  value: "java-it-service-account"
+  key: "{{ partial_key }}"
+  value: "{{ partial_value }}"
 }
+{% endfor -%}
+{%- endif -%}

synthtool/gcp/templates/java_library/defaults/integration-default.yaml

@@ -0,0 +1,9 @@
+# default env_vars in .kokoro/nightly/integration.cfg
+kokoro_nightly_env_vars:
+  TRAMPOLINE_IMAGE: gcr.io/cloud-devrel-kokoro-resources/java8
+  JOB_TYPE: integration
+  GCLOUD_PROJECT: java-docs-samples-testing
+  GOOGLE_CLOUD_PROJECT: java-docs-samples-testing
+  ENABLE_FLAKYBOT: true
+  GOOGLE_APPLICATION_CREDENTIALS: secret_manager/java-it-service-account
+  SECRET_MANAGER_KEYS: java-it-service-account

synthtool/languages/java.py

@@ -472,7 +472,10 @@ def _common_template_metadata() -> Dict[str, Any]:
 
 
 def common_templates(
-    excludes: List[str] = [], template_path: Optional[Path] = None, **kwargs
+    excludes: List[str] = [],
+    template_path: Optional[Path] = None,
+    partial_files: List[str] = None,
+    **kwargs,
 ) -> None:
     """Generate common templates for a Java Library
 
@@ -481,8 +484,12 @@ def common_templates(
     their expected location.
 
     Args:
-        excludes (List[str], optional): List of template paths to ignore
-        **kwargs: Additional options for CommonTemplates.java_library()
+        :param excludes: List of template paths to ignore
+        :param template_path:
+        :param partial_files: List of partial files to save customizations on
+        templates
+        :param kwargs: Additional options for CommonTemplates.java_library()
+
     """
     metadata = _common_template_metadata()
     kwargs["metadata"] = metadata
@@ -516,11 +523,14 @@ def common_templates(
         )
     )
 
-    templates = gcp.CommonTemplates(template_path=template_path).java_library(**kwargs)
+    templates = gcp.CommonTemplates(template_path=template_path).java_library(
+        partial_files, **kwargs
+    )
 
     # skip README generation on Kokoro (autosynth)
     if os.environ.get("KOKORO_ROOT") is not None:
-        # README.md is now synthesized separately. This prevents synthtool from deleting the
+        # README.md is now synthesized separately.
+        # This prevents synthtool from deleting the
         # README as it's no longer generated here.
         excludes.append("README.md")
 

tests/fixtures/java_templates/defaults_test/.repo-metadata.json

@@ -0,0 +1,18 @@
+{
+    "api_shortname": "cloudasset",
+    "name_pretty": "Cloud Asset Inventory",
+    "product_documentation": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview",
+    "api_reference": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview",
+    "api_description": "provides inventory services based on a time series database. This database keeps a five week history of Google Cloud asset metadata. The Cloud Asset Inventory export service allows you to export all asset metadata at a certain timestamp or export event change history during a timeframe.",
+    "client_documentation": "https://googleapis.dev/java/google-cloud-asset/latest/index.html",
+    "issue_tracker": "https://issuetracker.google.com/issues/new?component=187210&template=0",
+    "release_level": "stable",
+    "transport": "grpc",
+    "requires_billing": true,
+    "language": "java",
+    "repo": "googleapis/java-asset",
+    "repo_short": "java-asset",
+    "distribution_name": "com.google.cloud:google-cloud-asset",
+    "library_type": "GAPIC_AUTO",
+    "api_id": "cloudasset.googleapis.com"
+}

tests/fixtures/java_templates/defaults_test/integration-golden.cfg

@@ -0,0 +1,35 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+env_vars: {
+  key: "TRAMPOLINE_IMAGE"
+  value: "gcr.io/cloud-devrel-kokoro-resources/java8"
+}
+
+env_vars: {
+  key: "JOB_TYPE"
+  value: "integration"
+}
+
+env_vars: {
+  key: "GCLOUD_PROJECT"
+  value: "java-docs-samples-testing"
+}
+
+env_vars: {
+  key: "GOOGLE_CLOUD_PROJECT"
+  value: "java-docs-samples-testing"
+}
+
+env_vars: {
+  key: "ENABLE_FLAKYBOT"
+  value: "false"
+}
+
+env_vars: {
+  key: "GOOGLE_APPLICATION_CREDENTIALS"
+  value: "secret_manager/java-it-service-account"
+}
+
+env_vars: {
+  key: "SECRET_MANAGER_KEYS"
+  value: "java-it-service-account"
+}

tests/fixtures/java_templates/partials_test/.kokoro/nightly/integration.cfg-partials.yaml

@@ -0,0 +1,3 @@
+kokoro_nightly_env_vars:
+  INTEGRATION_TEST_ARGS: -P bigtable-emulator-it
+  GCLOUD_PROJECT: gcloud-devel

tests/fixtures/java_templates/partials_test/.repo-metadata.json

@@ -0,0 +1,18 @@
+{
+    "api_shortname": "cloudasset",
+    "name_pretty": "Cloud Asset Inventory",
+    "product_documentation": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview",
+    "api_reference": "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview",
+    "api_description": "provides inventory services based on a time series database. This database keeps a five week history of Google Cloud asset metadata. The Cloud Asset Inventory export service allows you to export all asset metadata at a certain timestamp or export event change history during a timeframe.",
+    "client_documentation": "https://googleapis.dev/java/google-cloud-asset/latest/index.html",
+    "issue_tracker": "https://issuetracker.google.com/issues/new?component=187210&template=0",
+    "release_level": "stable",
+    "transport": "grpc",
+    "requires_billing": true,
+    "language": "java",
+    "repo": "googleapis/java-asset",
+    "repo_short": "java-asset",
+    "distribution_name": "com.google.cloud:google-cloud-asset",
+    "library_type": "GAPIC_AUTO",
+    "api_id": "cloudasset.googleapis.com"
+}

tests/fixtures/java_templates/partials_test/integration-golden.cfg

@@ -0,0 +1,40 @@
+# Format: //devtools/kokoro/config/proto/build.proto
+env_vars: {
+  key: "TRAMPOLINE_IMAGE"
+  value: "gcr.io/cloud-devrel-kokoro-resources/java8"
+}
+
+env_vars: {
+  key: "JOB_TYPE"
+  value: "integration"
+}
+
+env_vars: {
+  key: "GOOGLE_CLOUD_PROJECT"
+  value: "java-docs-samples-testing"
+}
+
+env_vars: {
+  key: "ENABLE_FLAKYBOT"
+  value: "false"
+}
+
+env_vars: {
+  key: "GOOGLE_APPLICATION_CREDENTIALS"
+  value: "secret_manager/java-it-service-account"
+}
+
+env_vars: {
+  key: "SECRET_MANAGER_KEYS"
+  value: "java-it-service-account"
+}
+
+env_vars: {
+  key: "INTEGRATION_TEST_ARGS"
+  value: "-P bigtable-emulator-it"
+}
+
+env_vars: {
+  key: "GCLOUD_PROJECT"
+  value: "gcloud-devel"
+}

tests/test_language_java.py

@@ -254,6 +254,27 @@ def test_release_please_handle_releases():
             )
 
 
+def test_defaults():
+    with util.copied_fixtures_dir(FIXTURES / "java_templates" / "defaults_test"):
+        java.common_templates(template_path=TEMPLATES_PATH)
+        assert os.path.isfile(".kokoro/nightly/integration.cfg")
+        assert_matches_golden(
+            "integration-golden.cfg", ".kokoro/nightly/integration.cfg"
+        )
+
+
+def test_merge_partials():
+    with util.copied_fixtures_dir(FIXTURES / "java_templates" / "partials_test"):
+        java.common_templates(
+            template_path=TEMPLATES_PATH,
+            partial_files=[".kokoro/nightly/integration.cfg-partials.yaml"],
+        )
+        assert os.path.isfile(".kokoro/nightly/integration.cfg")
+        assert_matches_golden(
+            "integration-golden.cfg", ".kokoro/nightly/integration.cfg"
+        )
+
+
 def assert_matches_golden(expected, actual):
     matching_lines = 0
     with open(actual, "rt") as fp: