From acdde47445916dd306ce8b91489fab45c9c2ef50 Mon Sep 17 00:00:00 2001 From: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> Date: Thu, 25 Jan 2024 16:31:32 +0000 Subject: [PATCH] feat: move new client script (#2333) * feat: add new-client.py * refactor code * clone googleapis * add build parser * parse inputs from BUILD * find versioned dir within proto_path * fix postprocessing, use google-java-format * migirate shell scripts * change depth * add generate_gapic_bom.sh * process root pom * only generating gapic-libraries-bom if it already exists * add consolidate_config.sh * add generation dir * use variable for script dir * add set_parent_pom.sh * add apply_current_versions.sh * add click option for versions.txt * change dir * change find command * add readme_update.sh * change script path * refactor * code refactor * add generator version * use GitPython to checkout googleapis * code refactor * use versions.txt in the monorepo * add comments * change year * change copyright year * remove apply_current_versions.sh * add __pychche__ to gitignore * parse generator version from WORKSPACE * remove readme_update.sh * remove update_owlbot_postprocessor_config.sh * remove delete_non_generated_samples.sh * remove set_parent_pom.sh * add destination_name * restore format plugin * remove gitignore * remove consolidate_config.sh * change output directory --------- Co-authored-by: diegomarquezp --- .gitignore | 3 + .../new_client/client_inputs.py | 157 +++++++ .../get_generator_version_from_workspace.sh | 2 + library_generation/new_client/new-client.py | 421 ++++++++++++++++++ library_generation/new_client/requirements.in | 8 + .../new_client/requirements.txt | 232 ++++++++++ library_generation/new_client/templates.py | 33 ++ .../new_client/templates/owlbot.py.j2 | 28 ++ .../templates/owlbot.yaml.monorepo.j2 | 36 ++ library_generation/owlbot/bin/entrypoint.sh | 58 +-- .../owlbot/bin/format_source.sh | 53 +++ library_generation/owlbot/src/fix-poms.py | 51 ++- .../owlbot/templates/poms/bom_pom.xml.j2 | 13 +- .../owlbot/templates/poms/cloud_pom.xml.j2 | 10 +- .../owlbot/templates/poms/grpc_pom.xml.j2 | 2 + .../owlbot/templates/poms/parent_pom.xml.j2 | 10 +- .../owlbot/templates/poms/proto_pom.xml.j2 | 2 + library_generation/postprocess_library.sh | 10 - .../generate_gapic_bom.sh | 60 +++ .../generate_root_pom.sh | 13 + 20 files changed, 1127 insertions(+), 75 deletions(-) create mode 100644 library_generation/new_client/client_inputs.py create mode 100755 library_generation/new_client/get_generator_version_from_workspace.sh create mode 100644 library_generation/new_client/new-client.py create mode 100644 library_generation/new_client/requirements.in create mode 100644 library_generation/new_client/requirements.txt create mode 100644 library_generation/new_client/templates.py create mode 100644 library_generation/new_client/templates/owlbot.py.j2 create mode 100644 library_generation/new_client/templates/owlbot.yaml.monorepo.j2 create mode 100755 library_generation/owlbot/bin/format_source.sh create mode 100755 library_generation/repo-level-postprocess/generate_gapic_bom.sh create mode 100755 library_generation/repo-level-postprocess/generate_root_pom.sh diff --git a/.gitignore b/.gitignore index f98cac050a..3da2d8a7d2 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,9 @@ target/ *.iml +# Python +**/__pycache__/ + # library generation output/ library_generation/output/ diff --git a/library_generation/new_client/client_inputs.py b/library_generation/new_client/client_inputs.py new file mode 100644 index 0000000000..3106fe5210 --- /dev/null +++ b/library_generation/new_client/client_inputs.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path +import re + +proto_library_pattern = r""" +proto_library_with_info\( +(.*?) +\) +""" +gapic_pattern = r""" +java_gapic_library\( +(.*?) +\) +""" +assembly_pattern = r""" +java_gapic_assembly_gradle_pkg\( +(.*?) +\) +""" +resource_pattern = r"//google/cloud:common_resources_proto" +location_pattern = r"//google/cloud/location:location_proto" +iam_pattern = r"//google/iam/v1:iam_policy_proto" +transport_pattern = r"transport = \"(.*?)\"" +rest_pattern = r"rest_numeric_enums = True" +gapic_yaml_pattern = r"gapic_yaml = \"(.*?)\"" +service_config_pattern = r"grpc_service_config = \"(.*?)\"" +service_yaml_pattern = r"service_yaml = \"(.*?)\"" +include_samples_pattern = r"include_samples = True" + + +class ClientInput: + """ + A data class containing inputs to invoke generate_library.sh to generate + a GAPIC library. + """ + def __init__( + self, + proto_only="true", + additional_protos="google/cloud/common_resources.proto", + transport="", + rest_numeric_enum="", + gapic_yaml="", + service_config="", + service_yaml="", + include_samples="true", + ): + self.proto_only = proto_only + self.additional_protos = additional_protos + self.transport = transport + self.rest_numeric_enum = rest_numeric_enum + self.gapic_yaml = gapic_yaml + self.service_config = service_config + self.service_yaml = service_yaml + self.include_samples = include_samples + + +def parse( + build_path: Path, + versioned_path: str, +) -> ClientInput: + """ + Utility function to parse inputs of generate_library.sh from BUILD.bazel. + :param build_path: the file path of BUILD.bazel + :param versioned_path: a versioned path in googleapis repository, e.g., + google/cloud/asset/v1. + :return: an ClientInput object. + """ + with open(f"{build_path}/BUILD.bazel") as build: + content = build.read() + + proto_library_target = re.compile( + proto_library_pattern, re.DOTALL | re.VERBOSE + ).findall(content)[0] + additional_protos = __parse_additional_protos(proto_library_target) + gapic_target = re.compile(gapic_pattern, re.DOTALL | re.VERBOSE)\ + .findall(content) + assembly_target = re.compile(assembly_pattern, re.DOTALL | re.VERBOSE)\ + .findall(content) + include_samples = __parse_include_samples(assembly_target[0]) + if len(gapic_target) == 0: + return ClientInput( + include_samples=include_samples + ) + + transport = __parse_transport(gapic_target[0]) + rest_numeric_enum = __parse_rest_numeric_enums(gapic_target[0]) + gapic_yaml = __parse_gapic_yaml(gapic_target[0], versioned_path) + service_config = __parse_service_config(gapic_target[0], versioned_path) + service_yaml = __parse_service_yaml(gapic_target[0], versioned_path) + + return ClientInput( + proto_only="false", + additional_protos=additional_protos, + transport=transport, + rest_numeric_enum=rest_numeric_enum, + gapic_yaml=gapic_yaml, + service_config=service_config, + service_yaml=service_yaml, + include_samples=include_samples, + ) + + +def __parse_additional_protos(proto_library_target: str) -> str: + res = [" "] + if len(re.findall(resource_pattern, proto_library_target)) != 0: + res.append("google/cloud/common_resources.proto") + if len(re.findall(location_pattern, proto_library_target)) != 0: + res.append("google/cloud/location/locations.proto") + if len(re.findall(iam_pattern, proto_library_target)) != 0: + res.append("google/iam/v1/iam_policy.proto") + return " ".join(res) + + +def __parse_transport(gapic_target: str) -> str: + transport = re.findall(transport_pattern, gapic_target) + return transport[0] if len(transport) != 0 else "grpc" + + +def __parse_rest_numeric_enums(gapic_target: str) -> str: + rest_numeric_enums = re.findall(rest_pattern, gapic_target) + return "true" if len(rest_numeric_enums) != 0 else "false" + + +def __parse_gapic_yaml(gapic_target: str, versioned_path: str) -> str: + gapic_yaml = re.findall(gapic_yaml_pattern, gapic_target) + return f"{versioned_path}/{gapic_yaml[0]}" if len(gapic_yaml) != 0 else "" + + +def __parse_service_config(gapic_target: str, versioned_path: str) -> str: + service_config = re.findall(service_config_pattern, gapic_target) + return f"{versioned_path}/{service_config[0]}" if len(service_config) != 0 \ + else "" + + +def __parse_service_yaml(gapic_target: str, versioned_path: str) -> str: + service_yaml = re.findall(service_yaml_pattern, gapic_target) + return f"{versioned_path}/{service_yaml[0]}" if len(service_yaml) != 0 \ + else "" + + +def __parse_include_samples(assembly_target: str) -> str: + include_samples = re.findall(include_samples_pattern, assembly_target) + return "true" if len(include_samples) != 0 else "false" diff --git a/library_generation/new_client/get_generator_version_from_workspace.sh b/library_generation/new_client/get_generator_version_from_workspace.sh new file mode 100755 index 0000000000..0d8cac1f25 --- /dev/null +++ b/library_generation/new_client/get_generator_version_from_workspace.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +curl --silent 'https://raw.githubusercontent.com/googleapis/googleapis/master/WORKSPACE' | perl -nle 'print $1 if m/_gapic_generator_java_version\s+=\s+\"(.+)\"/' \ No newline at end of file diff --git a/library_generation/new_client/new-client.py b/library_generation/new_client/new-client.py new file mode 100644 index 0000000000..5b69f335c8 --- /dev/null +++ b/library_generation/new_client/new-client.py @@ -0,0 +1,421 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import click +import templates +from git import Repo +from client_inputs import parse +import shutil + + +@click.group(invoke_without_command=False) +@click.pass_context +@click.version_option(message="%(version)s") +def main(ctx): + pass + + +@main.command() +@click.option( + "--api_shortname", + required=True, + type=str, + prompt="Service name? (e.g. automl)", + help="Name for the new directory name and (default) artifact name" +) +@click.option( + "--name-pretty", + required=True, + type=str, + prompt="Pretty name? (e.g. 'Cloud AutoML')", + help="The human-friendly name that appears in README.md" +) +@click.option( + "--product-docs", + required=True, + type=str, + prompt="Product Documentation URL", + help="Documentation URL that appears in README.md" +) +@click.option( + "--api-description", + required=True, + type=str, + prompt="Description for README. The first sentence is prefixed by the " + "pretty name", + help="Description that appears in README.md" +) +@click.option( + "--release-level", + type=click.Choice(["stable", "preview"]), + default="preview", + show_default=True, + help="A label that appears in repo-metadata.json. The first library " + "generation is always 'preview'." +) +@click.option( + "--transport", + type=click.Choice(["grpc", "http", "both"]), + default="grpc", + show_default=True, + help="A label that appears in repo-metadata.json" +) +@click.option("--language", type=str, default="java", show_default=True) +@click.option( + "--distribution-name", + type=str, + help="Maven coordinates of the generated library. By default it's " + "com.google.cloud:google-cloud-" +) +@click.option( + "--api-id", + type=str, + help="The value of the apiid parameter used in README.md It has link to " + "https://console.cloud.google.com/flows/enableapi?apiid=" +) +@click.option( + "--requires-billing", + type=bool, + default=True, + show_default=True, + help="Based on this value, README.md explains whether billing setup is " + "needed or not." +) +@click.option( + "--destination-name", + type=str, + default=None, + help="The directory name of the new library. By default it's " + "java-" +) +@click.option( + "--proto-path", + required=True, + type=str, + default=None, + help="Path to proto file from the root of the googleapis repository to the" + "directory that contains the proto files (without the version)." + "For example, to generate the library for 'google/maps/routing/v2', " + "then you specify this value as 'google/maps/routing'" +) +@click.option( + "--cloud-api", + type=bool, + default=True, + show_default=True, + help="If true, the artifact ID of the library is 'google-cloud-'; " + "otherwise 'google-'" +) +@click.option( + "--group-id", + type=str, + default="com.google.cloud", + show_default=True, + help="The group ID of the artifact when distribution name is not set" +) +@click.option( + "--library-type", + type=str, + default="GAPIC_AUTO", + show_default=True, + help="A label that appear in repo-metadata.json to tell how the library is " + "maintained or generated" +) +@click.option( + "--googleapis-url", + type=str, + default="https://github.com/googleapis/googleapis.git", + show_default=True, + help="The URL of the repository that has proto service definition" +) +@click.option( + "--rest-docs", + type=str, + help="If it exists, link to the REST Documentation for a service" +) +@click.option( + "--rpc-docs", + type=str, + help="If it exists, link to the RPC Documentation for a service" +) +@click.option( + "--split-repo", + type=bool, + default=False, + help="Whether generating a library into a split repository" +) +def generate( + api_shortname, + name_pretty, + product_docs, + api_description, + release_level, + distribution_name, + api_id, + requires_billing, + transport, + language, + destination_name, + proto_path, + cloud_api, + group_id, + library_type, + googleapis_url, + rest_docs, + rpc_docs, + split_repo, +): + cloud_prefix = "cloud-" if cloud_api else "" + + output_name = destination_name if destination_name else api_shortname + if distribution_name is None: + distribution_name = f"{group_id}:google-{cloud_prefix}{output_name}" + + distribution_name_short = re.split(r"[:\/]", distribution_name)[-1] + + if api_id is None: + api_id = f"{api_shortname}.googleapis.com" + + if not product_docs.startswith("https"): + sys.exit("product_docs must starts with 'https://'") + + client_documentation = ( + f"https://cloud.google.com/{language}/docs/reference/{distribution_name_short}/latest/overview" + ) + + if api_shortname == "": + sys.exit("api_shortname is empty") + + repo = "googleapis/google-cloud-java" + if split_repo: + repo = f"{language}-{output_name}" + + repo_metadata = { + "api_shortname": api_shortname, + "name_pretty": name_pretty, + "product_documentation": product_docs, + "api_description": api_description, + "client_documentation": client_documentation, + "release_level": release_level, + "transport": transport, + "language": language, + "repo": f"{repo}", + "repo_short": f"{language}-{output_name}", + "distribution_name": distribution_name, + "api_id": api_id, + "library_type": library_type, + } + if requires_billing: + repo_metadata["requires_billing"] = True + + if rest_docs: + repo_metadata["rest_documentation"] = rest_docs + + if rpc_docs: + repo_metadata["rpc_documentation"] = rpc_docs + # Initialize workdir + workdir = Path(f"{sys.path[0]}/../../output/java-{output_name}").resolve() + if os.path.isdir(workdir): + sys.exit( + "Couldn't create the module because " + f"the module {workdir} already exists. In Java client library " + "generation, a new API version of an existing module does not " + "require new-client.py invocation. " + "See go/yoshi-java-new-client#adding-a-new-service-version-by-owlbot." + ) + print(f"Creating a new module {workdir}") + os.makedirs(workdir, exist_ok=False) + # write .repo-metadata.json file + with open(workdir / ".repo-metadata.json", "w") as fp: + json.dump(repo_metadata, fp, indent=2) + + template_excludes = [ + ".github/*", + ".kokoro/*", + "samples/*", + "CODE_OF_CONDUCT.md", + "CONTRIBUTING.md", + "LICENSE", + "SECURITY.md", + "java.header", + "license-checks.xml", + "renovate.json", + ".gitignore" + ] + # create owlbot.py + templates.render( + template_name="owlbot.py.j2", + output_name=str(workdir / "owlbot.py"), + should_include_templates=True, + template_excludes=template_excludes, + ) + + # In monorepo, .OwlBot.yaml needs to be in the directory of the module. + owlbot_yaml_location_from_module = ".OwlBot.yaml" + # create owlbot config + templates.render( + template_name="owlbot.yaml.monorepo.j2", + output_name=str(workdir / owlbot_yaml_location_from_module), + artifact_name=distribution_name_short, + proto_path=proto_path, + module_name=f"java-{output_name}", + api_shortname=api_shortname + ) + + print(f"Pulling proto from {googleapis_url}") + output_dir = Path(f"{sys.path[0]}/../../output").resolve() + __sparse_clone( + remote_url=googleapis_url, + dest=output_dir, + ) + # Find a versioned directory within proto_path + # We only need to generate one version of the library as OwlBot + # will copy other versions from googleapis-gen. + version = __find_version( + Path(f"{sys.path[0]}/../../output/{proto_path}").resolve() + ) + versioned_proto_path = f"{proto_path}/{version}" + print(f"Generating from {versioned_proto_path}") + # parse BUILD.bazel in proto_path + client_input = parse( + build_path=Path(f"{sys.path[0]}/../../output/{versioned_proto_path}") + .resolve(), + versioned_path=versioned_proto_path, + ) + repo_root_dir = Path(f"{sys.path[0]}/../../").resolve() + generator_version = subprocess.check_output( + ["library_generation/new_client/get_generator_version_from_workspace.sh"], + cwd=repo_root_dir + ).strip() + print(f"Generator version: {generator_version}") + # run generate_library.sh + subprocess.check_call([ + "library_generation/generate_library.sh", + "-p", + versioned_proto_path, + "-d", + f"java-{output_name}", + "--gapic_generator_version", + generator_version, + "--protobuf_version", + "23.2", + "--proto_only", + client_input.proto_only, + "--gapic_additional_protos", + client_input.additional_protos, + "--transport", + client_input.transport, + "--rest_numeric_enums", + client_input.rest_numeric_enum, + "--gapic_yaml", + client_input.gapic_yaml, + "--service_config", + client_input.service_config, + "--service_yaml", + client_input.service_yaml, + "--include_samples", + client_input.include_samples, + "--versions_file", + f"{repo_root_dir}/versions.txt"], + cwd=repo_root_dir + ) + + # Move generated module to repo root. + __move_modules( + source=output_dir, + dest=repo_root_dir, + name_prefix="java-" + ) + + # Repo level post process + script_dir = "library_generation/repo-level-postprocess" + + print("Regenerating root pom.xml") + subprocess.check_call( + [ + f"{script_dir}/generate_root_pom.sh", + f"{output_dir}" + ], + cwd=repo_root_dir, + ) + + if not split_repo: + print("Regenerating the GAPIC BOM") + subprocess.check_call( + [ + f"{script_dir}/generate_gapic_bom.sh", + f"{output_dir}" + ], + cwd=repo_root_dir, + ) + + print("Deleting temp files") + subprocess.check_call( + [ + "rm", + "-rf", + f"{output_dir}" + ], + cwd=repo_root_dir + ) + + print(f"Prepared new library in {workdir}") + print(f"Please create a pull request:\n" + f" $ git checkout -b new_module_java-{output_name}\n" + f" $ git add .\n" + f" $ git commit -m 'feat: [{api_shortname}] new module for {api_shortname}'\n" + f" $ gh pr create --title 'feat: [{api_shortname}] new module for {api_shortname}'") + + +def __sparse_clone( + remote_url: str, + dest: Path, + commit_hash: str = "master", +): + local_repo = Repo.init(dest) + origin = local_repo.create_remote( + name="origin", + url=remote_url + ) + + origin.fetch() + git = local_repo.git() + git.checkout(f"origin/{commit_hash}", "--", "google", "grafeas") + + +def __find_version(proto_path: Path) -> str: + for child in proto_path.iterdir(): + if child.is_dir() and re.search(r"v[1-9]", child.name) is not None: + return child.name + return "" + + +def __move_modules( + source: Path, + dest: Path, + name_prefix: str +) -> None: + for folder in source.iterdir(): + if folder.is_dir() and folder.name.startswith(name_prefix): + shutil.move(folder, dest) + + +if __name__ == "__main__": + main() diff --git a/library_generation/new_client/requirements.in b/library_generation/new_client/requirements.in new file mode 100644 index 0000000000..2ff144604c --- /dev/null +++ b/library_generation/new_client/requirements.in @@ -0,0 +1,8 @@ +attr +attrs +black +click +jinja2 +lxml +typing +GitPython diff --git a/library_generation/new_client/requirements.txt b/library_generation/new_client/requirements.txt new file mode 100644 index 0000000000..1012323e89 --- /dev/null +++ b/library_generation/new_client/requirements.txt @@ -0,0 +1,232 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --generate-hashes library_generation/new_client/requirements.in +# +attr==0.3.2 \ + --hash=sha256:1ceebca768181cdcce9827611b1d728e592be5d293911539ea3d0b0bfa1146f4 \ + --hash=sha256:4f4bffeea8c27387bde446675a7ac24f3b8fea1075f12d849b5f5c5181fc8336 + # via -r library_generation/new_client/requirements.in +attrs==23.2.0 \ + --hash=sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30 \ + --hash=sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1 + # via -r library_generation/new_client/requirements.in +black==23.12.1 \ + --hash=sha256:0808494f2b2df923ffc5723ed3c7b096bd76341f6213989759287611e9837d50 \ + --hash=sha256:1fa88a0f74e50e4487477bc0bb900c6781dbddfdfa32691e780bf854c3b4a47f \ + --hash=sha256:25e57fd232a6d6ff3f4478a6fd0580838e47c93c83eaf1ccc92d4faf27112c4e \ + --hash=sha256:2d9e13db441c509a3763a7a3d9a49ccc1b4e974a47be4e08ade2a228876500ec \ + --hash=sha256:3e1b38b3135fd4c025c28c55ddfc236b05af657828a8a6abe5deec419a0b7055 \ + --hash=sha256:3fa4be75ef2a6b96ea8d92b1587dd8cb3a35c7e3d51f0738ced0781c3aa3a5a3 \ + --hash=sha256:4ce3ef14ebe8d9509188014d96af1c456a910d5b5cbf434a09fef7e024b3d0d5 \ + --hash=sha256:4f0031eaa7b921db76decd73636ef3a12c942ed367d8c3841a0739412b260a54 \ + --hash=sha256:602cfb1196dc692424c70b6507593a2b29aac0547c1be9a1d1365f0d964c353b \ + --hash=sha256:6d1bd9c210f8b109b1762ec9fd36592fdd528485aadb3f5849b2740ef17e674e \ + --hash=sha256:78baad24af0f033958cad29731e27363183e140962595def56423e626f4bee3e \ + --hash=sha256:8d4df77958a622f9b5a4c96edb4b8c0034f8434032ab11077ec6c56ae9f384ba \ + --hash=sha256:97e56155c6b737854e60a9ab1c598ff2533d57e7506d97af5481141671abf3ea \ + --hash=sha256:9c4352800f14be5b4864016882cdba10755bd50805c95f728011bcb47a4afd59 \ + --hash=sha256:a4d6a9668e45ad99d2f8ec70d5c8c04ef4f32f648ef39048d010b0689832ec6d \ + --hash=sha256:a920b569dc6b3472513ba6ddea21f440d4b4c699494d2e972a1753cdc25df7b0 \ + --hash=sha256:ae76c22bde5cbb6bfd211ec343ded2163bba7883c7bc77f6b756a1049436fbb9 \ + --hash=sha256:b18fb2ae6c4bb63eebe5be6bd869ba2f14fd0259bda7d18a46b764d8fb86298a \ + --hash=sha256:c04b6d9d20e9c13f43eee8ea87d44156b8505ca8a3c878773f68b4e4812a421e \ + --hash=sha256:c88b3711d12905b74206227109272673edce0cb29f27e1385f33b0163c414bba \ + --hash=sha256:dd15245c8b68fe2b6bd0f32c1556509d11bb33aec9b5d0866dd8e2ed3dba09c2 \ + --hash=sha256:e0aaf6041986767a5e0ce663c7a2f0e9eaf21e6ff87a5f95cbf3675bfd4c41d2 + # via -r library_generation/new_client/requirements.in +click==8.1.7 \ + --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ + --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de + # via + # -r library_generation/new_client/requirements.in + # black +gitdb==4.0.11 \ + --hash=sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4 \ + --hash=sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b + # via gitpython +gitpython==3.1.40 \ + --hash=sha256:22b126e9ffb671fdd0c129796343a02bf67bf2994b35449ffc9321aa755e18a4 \ + --hash=sha256:cf14627d5a8049ffbf49915732e5eddbe8134c3bdb9d476e6182b676fc573f8a + # via -r library_generation/new_client/requirements.in +jinja2==3.1.2 \ + --hash=sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 \ + --hash=sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61 + # via -r library_generation/new_client/requirements.in +lxml==5.0.0 \ + --hash=sha256:016de3b29a262655fc3d2075dc1b2611f84f4c3d97a71d579c883d45e201eee4 \ + --hash=sha256:0326e9b8176ea77269fb39e7af4010906e73e9496a9f8eaf06d253b1b1231ceb \ + --hash=sha256:03290e2f714f2e7431c8430c08b48167f657da7bc689c6248e828ff3c66d5b1b \ + --hash=sha256:049fef98d02513c34f5babd07569fc1cf1ed14c0f2fbff18fe72597f977ef3c2 \ + --hash=sha256:07a900735bad9af7be3085480bf384f68ed5580ba465b39a098e6a882c060d6b \ + --hash=sha256:0d277d4717756fe8816f0beeff229cb72f9dd02a43b70e1d3f07c8efadfb9fe1 \ + --hash=sha256:173bcead3af5d87c7bca9a030675073ddaad8e0a9f0b04be07cd9390453e7226 \ + --hash=sha256:1ef0793e1e2dd221fce7c142177008725680f7b9e4a184ab108d90d5d3ab69b7 \ + --hash=sha256:21af2c3862db6f4f486cddf73ec1157b40d5828876c47cd880edcbad8240ea1b \ + --hash=sha256:2219cbf790e701acf9a21a31ead75f983e73daf0eceb9da6990212e4d20ebefe \ + --hash=sha256:2992591e2294bb07faf7f5f6d5cb60710c046404f4bfce09fb488b85d2a8f58f \ + --hash=sha256:3663542aee845129a981889c19b366beab0b1dadcf5ca164696aabfe1aa51667 \ + --hash=sha256:3e6cbb68bf70081f036bfc018649cf4b46c4e7eaf7860a277cae92dee2a57f69 \ + --hash=sha256:3f908afd0477cace17f941d1b9cfa10b769fe1464770abe4cfb3d9f35378d0f8 \ + --hash=sha256:3ffa066db40b0347e48334bd4465de768e295a3525b9a59831228b5f4f93162d \ + --hash=sha256:405e3760f83a8ba3bdb6e622ec79595cdc20db916ce37377bbcb95b5711fa4ca \ + --hash=sha256:44fa9afd632210f1eeda51cf284ed8dbab0c7ec8b008dd39ba02818e0e114e69 \ + --hash=sha256:4786b0af7511ea614fd86407a52a7bc161aa5772d311d97df2591ed2351de768 \ + --hash=sha256:4a45a278518e4308865c1e9dbb2c42ce84fb154efb03adeb16fdae3c1687c7c9 \ + --hash=sha256:4b9d5b01900a760eb3acf6cef50aead4ef2fa79e7ddb927084244e41dfe37b65 \ + --hash=sha256:4e69c36c8618707a90ed3fb6f48a6cc9254ffcdbf7b259e439a5ae5fbf9c5206 \ + --hash=sha256:52a9ab31853d3808e7cf0183b3a5f7e8ffd622ea4aee1deb5252dbeaefd5b40d \ + --hash=sha256:52c0acc2f29b0a204efc11a5ed911a74f50a25eb7d7d5069c2b1fd3b3346ce11 \ + --hash=sha256:5382612ba2424cea5d2c89e2c29077023d8de88f8d60d5ceff5f76334516df9e \ + --hash=sha256:581a78f299a9f5448b2c3aea904bfcd17c59bf83016d221d7f93f83633bb2ab2 \ + --hash=sha256:583c0e15ae06adc81035346ae2abb2e748f0b5197e7740d8af31222db41bbf7b \ + --hash=sha256:59cea9ba1c675fbd6867ca1078fc717a113e7f5b7644943b74137b7cc55abebf \ + --hash=sha256:5b39f63edbe7e018c2ac1cf0259ee0dd2355274e8a3003d404699b040782e55e \ + --hash=sha256:5eff173f0ff408bfa578cbdafd35a7e0ca94d1a9ffe09a8a48e0572d0904d486 \ + --hash=sha256:5fb988e15378d6e905ca8f60813950a0c56da9469d0e8e5d8fe785b282684ec5 \ + --hash=sha256:6507c58431dbd95b50654b3313c5ad54f90e54e5f2cdacf733de61eae478eec5 \ + --hash=sha256:6a2de85deabf939b0af89e2e1ea46bfb1239545e2da6f8ac96522755a388025f \ + --hash=sha256:6a5501438dd521bb7e0dde5008c40c7bfcfaafaf86eccb3f9bd27509abb793da \ + --hash=sha256:6bba06d8982be0f0f6432d289a8d104417a0ab9ed04114446c4ceb6d4a40c65d \ + --hash=sha256:70ab4e02f7aa5fb4131c8b222a111ce7676f3767e36084fba3a4e7338dc82dcd \ + --hash=sha256:7188495c1bf71bfda87d78ed50601e72d252119ce11710d6e71ff36e35fea5a0 \ + --hash=sha256:71a7cee869578bc17b18050532bb2f0bc682a7b97dda77041741a1bd2febe6c7 \ + --hash=sha256:73bfab795d354aaf2f4eb7a5b0db513031734fd371047342d5803834ce19ec18 \ + --hash=sha256:766868f729f3ab84125350f1a0ea2594d8b1628a608a574542a5aff7355b9941 \ + --hash=sha256:77b73952534967a4497d9e4f26fbeebfba19950cbc66b7cc3a706214429d8106 \ + --hash=sha256:78d6d8e5b54ed89dc0f0901eaaa579c384ad8d59fa43cc7fb06e9bb89115f8f4 \ + --hash=sha256:793be9b4945c2dfd69828fb5948d7d9569b78e0599e4a2e88d92affeb0ff3aa3 \ + --hash=sha256:7ba26a7dc929a1b3487d51bbcb0099afed2fc06e891b82845c8f37a2d7d7fbbd \ + --hash=sha256:7df433d08d4587dc3932f7fcfc3194519a6824824104854e76441fd3bc000d29 \ + --hash=sha256:80209b31dd3908bc5b014f540fd192c97ea52ab179713a730456c5baf7ce80c1 \ + --hash=sha256:8134d5441d1ed6a682e3de3d7a98717a328dce619ee9c4c8b3b91f0cb0eb3e28 \ + --hash=sha256:81509dffd8aba3bdb43e90cbd218c9c068a1f4047d97bc9546b3ac9e3a4ae81d \ + --hash=sha256:88f559f8beb6b90e41a7faae4aca4c8173a4819874a9bf8e74c8d7c1d51f3162 \ + --hash=sha256:894c5f71186b410679aaab5774543fcb9cbabe8893f0b31d11cf28a0740e80be \ + --hash=sha256:8cc0a951e5616ac626f7036309c41fb9774adcd4aa7db0886463da1ce5b65edb \ + --hash=sha256:8ce8b468ab50f9e944719d1134709ec11fe0d2840891a6cae369e22141b1094c \ + --hash=sha256:904d36165848b59c4e04ae5b969072e602bd987485076fca8ec42c6cd7a7aedc \ + --hash=sha256:96095bfc0c02072fc89afa67626013a253596ea5118b8a7f4daaae049dafa096 \ + --hash=sha256:980ba47c8db4b9d870014c7040edb230825b79017a6a27aa54cdb6fcc02d8cc0 \ + --hash=sha256:992029258ed719f130d5a9c443d142c32843046f1263f2c492862b2a853be570 \ + --hash=sha256:99cad5c912f359e59e921689c04e54662cdd80835d80eeaa931e22612f515df7 \ + --hash=sha256:9b59c429e1a2246da86ae237ffc3565efcdc71c281cd38ca8b44d5fb6a3b993a \ + --hash=sha256:9ca498f8554a09fbc3a2f8fc4b23261e07bc27bef99b3df98e2570688033f6fc \ + --hash=sha256:9cd3d6c2c67d4fdcd795e4945e2ba5434909c96640b4cc09453bd0dc7e8e1bac \ + --hash=sha256:a85136d0ee18a41c91cc3e2844c683be0e72e6dda4cb58da9e15fcaef3726af7 \ + --hash=sha256:ac21aace6712472e77ea9dfc38329f53830c4259ece54c786107105ebb069053 \ + --hash=sha256:aebd8fd378e074b22e79cad329dcccd243c40ff1cafaa512d19276c5bb9554e1 \ + --hash=sha256:affdd833f82334fdb10fc9a1c7b35cdb5a86d0b672b4e14dd542e1fe7bcea894 \ + --hash=sha256:b6d4e148edee59c2ad38af15810dbcb8b5d7b13e5de3509d8cf3edfe74c0adca \ + --hash=sha256:bb58e8f4b2cfe012cd312239b8d5139995fe8f5945c7c26d5fbbbb1ddb9acd47 \ + --hash=sha256:bfdc4668ac56687a89ca3eca44231144a2e9d02ba3b877558db74ba20e2bd9fa \ + --hash=sha256:c1249aa4eaced30b59ecf8b8cae0b1ccede04583c74ca7d10b6f8bbead908b2c \ + --hash=sha256:c7cfb6af73602c8d288581df8a225989d7e9d5aab0a174be0e19fcfa800b6797 \ + --hash=sha256:c7fe19abb3d3c55a9e65d289b12ad73b3a31a3f0bda3c539a890329ae9973bd6 \ + --hash=sha256:c8954da15403db1acfc0544b3c3f963a6ef4e428283ab6555e3e298bbbff1cf6 \ + --hash=sha256:c90c593aa8dd57d5dab0ef6d7d64af894008971d98e6a41b320fdd75258fbc6e \ + --hash=sha256:cb564bbe55ff0897d9cf1225041a44576d7ae87f06fd60163544c91de2623d3f \ + --hash=sha256:cfa8a4cdc3765574b7fd0c7cfa5fbd1e2108014c9dfd299c679e5152bea9a55e \ + --hash=sha256:d1bb64646480c36a4aa1b6a44a5b6e33d0fcbeab9f53f1b39072cd3bb2c6243a \ + --hash=sha256:dac2733fe4e159b0aae0439db6813b7b1d23ff96d0b34c0107b87faf79208c4e \ + --hash=sha256:db40e85cffd22f7d65dcce30e85af565a66401a6ed22fc0c56ed342cfa4ffc43 \ + --hash=sha256:dd39ef87fd1f7bb5c4aa53454936e6135cbfe03fe3744e8218be193f9e4fef16 \ + --hash=sha256:de1a8b54170024cf1c0c2718c82412bca42cd82e390556e3d8031af9541b416f \ + --hash=sha256:e675a4b95208e74c34ac0751cc4bab9170e7728b61601fb0f4746892c2bb7e0b \ + --hash=sha256:e6bb39d91bf932e7520cb5718ae3c2f498052aca53294d5d59fdd9068fe1a7f2 \ + --hash=sha256:e8c63f5c7d87e7044880b01851ac4e863c3349e6f6b6ab456fe218d9346e816d \ + --hash=sha256:ea56825c1e23c9c8ea385a191dac75f9160477057285b88c88736d9305e6118f \ + --hash=sha256:ee60f33456ff34b2dd1d048a740a2572798356208e4c494301c931de3a0ab3a2 \ + --hash=sha256:f15844a1b93dcaa09c2b22e22a73384f3ae4502347c3881cfdd674e14ac04e21 \ + --hash=sha256:f298ac9149037d6a3d5c74991bded39ac46292520b9c7c182cb102486cc87677 \ + --hash=sha256:f30e697b6215e759d0824768b2c5b0618d2dc19abe6c67eeed2b0460f52470d1 \ + --hash=sha256:f92d73faa0b1a76d1932429d684b7ce95829e93c3eef3715ec9b98ab192c9d31 \ + --hash=sha256:fef10f27d6318d2d7c88680e113511ddecf09ee4f9559b3623b73ee89fa8f6cc + # via -r library_generation/new_client/requirements.in +markupsafe==2.1.3 \ + --hash=sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e \ + --hash=sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e \ + --hash=sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431 \ + --hash=sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686 \ + --hash=sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c \ + --hash=sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559 \ + --hash=sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc \ + --hash=sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb \ + --hash=sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939 \ + --hash=sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c \ + --hash=sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0 \ + --hash=sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4 \ + --hash=sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9 \ + --hash=sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575 \ + --hash=sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba \ + --hash=sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d \ + --hash=sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd \ + --hash=sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3 \ + --hash=sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00 \ + --hash=sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155 \ + --hash=sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac \ + --hash=sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52 \ + --hash=sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f \ + --hash=sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8 \ + --hash=sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b \ + --hash=sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007 \ + --hash=sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24 \ + --hash=sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea \ + --hash=sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198 \ + --hash=sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0 \ + --hash=sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee \ + --hash=sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be \ + --hash=sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2 \ + --hash=sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1 \ + --hash=sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707 \ + --hash=sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6 \ + --hash=sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c \ + --hash=sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58 \ + --hash=sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823 \ + --hash=sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779 \ + --hash=sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636 \ + --hash=sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c \ + --hash=sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad \ + --hash=sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee \ + --hash=sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc \ + --hash=sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2 \ + --hash=sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48 \ + --hash=sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7 \ + --hash=sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e \ + --hash=sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b \ + --hash=sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa \ + --hash=sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5 \ + --hash=sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e \ + --hash=sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb \ + --hash=sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9 \ + --hash=sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57 \ + --hash=sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc \ + --hash=sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc \ + --hash=sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2 \ + --hash=sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11 + # via jinja2 +mypy-extensions==1.0.0 \ + --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \ + --hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782 + # via black +packaging==23.2 \ + --hash=sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5 \ + --hash=sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7 + # via black +pathspec==0.12.1 \ + --hash=sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08 \ + --hash=sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712 + # via black +platformdirs==4.1.0 \ + --hash=sha256:11c8f37bcca40db96d8144522d925583bdb7a31f7b0e37e3ed4318400a8e2380 \ + --hash=sha256:906d548203468492d432bcb294d4bc2fff751bf84971fbb2c10918cc206ee420 + # via black +smmap==5.0.1 \ + --hash=sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62 \ + --hash=sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da + # via gitdb +typing==3.7.4.3 \ + --hash=sha256:1187fb9c82fd670d10aa07bbb6cfcfe4bdda42d6fab8d5134f04e8c4d0b71cc9 \ + --hash=sha256:283d868f5071ab9ad873e5e52268d611e851c870a2ba354193026f2dfb29d8b5 + # via -r library_generation/new_client/requirements.in diff --git a/library_generation/new_client/templates.py b/library_generation/new_client/templates.py new file mode 100644 index 0000000000..5b0282ce03 --- /dev/null +++ b/library_generation/new_client/templates.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from jinja2 import Environment, FileSystemLoader +import os +import pathlib + +root_directory = pathlib.Path( + os.path.realpath(os.path.dirname(os.path.realpath(__file__))) +) +print(root_directory) +jinja_env = Environment(loader=FileSystemLoader(str(root_directory / "templates"))) + + +def render(template_name: str, output_name: str, **kwargs): + template = jinja_env.get_template(template_name) + t = template.stream(kwargs) + directory = os.path.dirname(output_name) + if not os.path.isdir(directory): + os.makedirs(directory) + t.dump(str(output_name)) diff --git a/library_generation/new_client/templates/owlbot.py.j2 b/library_generation/new_client/templates/owlbot.py.j2 new file mode 100644 index 0000000000..84f070f35b --- /dev/null +++ b/library_generation/new_client/templates/owlbot.py.j2 @@ -0,0 +1,28 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import synthtool as s +{% if should_include_templates %}from synthtool.languages import java{% endif %} + + +for library in s.get_staging_dirs(): + # put any special-case replacements here + s.move(library) + +s.remove_staging_dirs() +{% if should_include_templates %}java.common_templates(monorepo=True, {% if template_excludes %}excludes=[ +{%- for exclude in template_excludes %} + "{{ exclude }}", +{%- endfor %} +]{% endif %}){% endif %} diff --git a/library_generation/new_client/templates/owlbot.yaml.monorepo.j2 b/library_generation/new_client/templates/owlbot.yaml.monorepo.j2 new file mode 100644 index 0000000000..3cfcc46aaf --- /dev/null +++ b/library_generation/new_client/templates/owlbot.yaml.monorepo.j2 @@ -0,0 +1,36 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{% if artifact_name %} +deep-remove-regex: +- "/{{ module_name }}/grpc-google-.*/src" +- "/{{ module_name }}/proto-google-.*/src" +- "/{{ module_name }}/google-.*/src" +- "/{{ module_name }}/samples/snippets/generated" + +deep-preserve-regex: +- "/{{ module_name }}/google-.*/src/test/java/com/google/cloud/.*/v.*/it/IT.*Test.java" + +deep-copy-regex: +- source: "/{{ proto_path }}/(v.*)/.*-java/proto-google-.*/src" + dest: "/owl-bot-staging/{{ module_name }}/$1/proto-{{ artifact_name }}-$1/src" +- source: "/{{ proto_path }}/(v.*)/.*-java/grpc-google-.*/src" + dest: "/owl-bot-staging/{{ module_name }}/$1/grpc-{{ artifact_name }}-$1/src" +- source: "/{{ proto_path }}/(v.*)/.*-java/gapic-google-.*/src" + dest: "/owl-bot-staging/{{ module_name }}/$1/{{ artifact_name }}/src" +- source: "/{{ proto_path }}/(v.*)/.*-java/samples/snippets/generated" + dest: "/owl-bot-staging/{{ module_name }}/$1/samples/snippets/generated" +{% endif %} + +api-name: {{ api_shortname }} diff --git a/library_generation/owlbot/bin/entrypoint.sh b/library_generation/owlbot/bin/entrypoint.sh index f483f98cfd..65e3a5fa2a 100755 --- a/library_generation/owlbot/bin/entrypoint.sh +++ b/library_generation/owlbot/bin/entrypoint.sh @@ -65,49 +65,17 @@ function processModule() { echo "...done" } -if [ "$(ls */.OwlBot.yaml|wc -l)" -gt 1 ];then - # Monorepo (googleapis/google-cloud-java) has multiple OwlBot.yaml config - # files in the modules. - echo "Processing monorepo" - if [ -d owl-bot-staging ]; then - # The content of owl-bot-staging is controlled by Owlbot.yaml files in - # each module in the monorepo - echo "Extracting contents from owl-bot-staging" - for module in owl-bot-staging/* ; do - if [ ! -d "$module" ]; then - continue - fi - # This relocation allows us continue to use owlbot.py without modification - # after monorepo migration. - mv "owl-bot-staging/$module" "$module/owl-bot-staging" - pushd "$module" - processModule - popd - done - rm -r owl-bot-staging - else - echo "In monorepo but no owl-bot-staging." \ - "Formatting changes in the last commit" - # Find the files that were touched by the last commit. - last_commit=$(git log -1 --format=%H) - # [A]dded, [C]reated, [M]odified, and [R]enamed - changed_files=$(git show --name-only --no-renames --diff-filter=ACMR \ - "${last_commit}") - changed_modules=$(echo "$changed_files" |grep -E '.java$' |cut -d '/' -f 1 \ - |sort -u) - for module in ${changed_modules}; do - if [ ! -f "$module/.OwlBot.yaml" ]; then - # Changes irrelevant to Owlbot-generated module (such as .github) do not - # need formatting - continue - fi - pushd "$module" - processModule - popd - done - fi -else - # Split repository - echo "Processing a split repo" - processModule +# monorepo folders have an .OwlBot.yaml file in the module folder (e.g. +# java-asset/.OwlBot.yaml), whereas HW libraries have the yaml in +# `.github/.OwlBot.yaml` +if [[ -f "$(pwd)/.OwlBot.yaml" ]]; then + monorepo="true" +fi + +if [[ "${monorepo}" == "true" ]]; then + mv owl-bot-staging/* temp + rm -rd owl-bot-staging/ + mv temp owl-bot-staging fi + +processModule diff --git a/library_generation/owlbot/bin/format_source.sh b/library_generation/owlbot/bin/format_source.sh new file mode 100755 index 0000000000..849e27f74f --- /dev/null +++ b/library_generation/owlbot/bin/format_source.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# Why OwlBot Java postprocessor does not use the formatter defined in pom.xml? +# It's because the postprocessor runs in a privileged (albeit limited) +# environment. We limit the risk of running somebody else's malicious Maven +# plugin code in the environment. + +# Find all the java files relative to the current directory and format them +# using google-java-format +list="$(find . -name '*.java' -not -path ".*/samples/snippets/generated/**/*" )" +scripts_root=$1 +tmpfile=$(mktemp) + +for file in $list; +do + if [[ $file =~ .*/samples/snippets/src/main/java/com/example/firestore/Quickstart.java ]]; + then + echo "File skipped formatting: $file" + elif [[ $file =~ .*/samples/snippets/src/.*/java/com/example/spanner/.*.java ]]; + then + echo "File skipped formatting: $file" + else + echo $file >> $tmpfile + fi +done + +# download the google-java-format tool +if [ ! -f "${scripts_root}/owlbot/google-java-format.jar" ]; then + echo 'downloading google-java-format' + java_format_version=$(cat "${scripts_root}/configuration/java-format-version") + wget -O "${scripts_root}/owlbot/google-java-format.jar" https://repo1.maven.org/maven2/com/google/googlejavaformat/google-java-format/${java_format_version}/google-java-format-${java_format_version}-all-deps.jar + +fi + +# format the source +cat $tmpfile | xargs java -jar "${scripts_root}/owlbot/google-java-format.jar" --replace + +rm $tmpfile diff --git a/library_generation/owlbot/src/fix-poms.py b/library_generation/owlbot/src/fix-poms.py index e4617a5085..87df4a46bb 100644 --- a/library_generation/owlbot/src/fix-poms.py +++ b/library_generation/owlbot/src/fix-poms.py @@ -22,6 +22,7 @@ import re from typing import List, Mapping from poms import module, templates +from pathlib import Path def load_versions(filename: str, default_group_id: str) -> Mapping[str, module.Module]: @@ -40,7 +41,7 @@ def load_versions(filename: str, default_group_id: str) -> Mapping[str, module.M group_id = ( default_group_id if artifact_id.startswith("google-") - else _proto_group_id(default_group_id) + else __proto_group_id(default_group_id) ) modules[artifact_id] = module.Module( group_id=group_id, @@ -282,7 +283,7 @@ def update_bom_pom(filename: str, modules: List[module.Module]): # https://github.com/googleapis/google-cloud-java/issues/9125 # However, some exceptions are com.google.area120 and com.google.analytics. # https://github.com/googleapis/google-cloud-java/issues/9304 -def _proto_group_id(main_artifact_group_id: str) -> str: +def __proto_group_id(main_artifact_group_id: str) -> str: prefix = "com.google" list_of_group_id = ["com.google.cloud", "com.google.area120", @@ -292,6 +293,18 @@ def _proto_group_id(main_artifact_group_id: str) -> str: return f"{prefix}.api.grpc" +def __get_monorepo_version(versions: str) -> str: + """ + Returns the current version of google-cloud-java in the given version file + :param versions: the versions.txt + :return: the current version of google-cloud-java + """ + with open(versions, "r") as f: + for line in f.readlines(): + if "google-cloud-java" in line: + return line.split(":")[-1].strip() + + def main(versions_file, monorepo): print(f"working directory: {os.getcwd()}") with open(".repo-metadata.json", "r") as fp: @@ -365,7 +378,7 @@ def main(versions_file, monorepo): for path in glob.glob("proto-google-*"): if not path in existing_modules: existing_modules[path] = module.Module( - group_id=_proto_group_id(group_id), + group_id=__proto_group_id(group_id), artifact_id=path, version=main_module.version, release_version=main_module.release_version, @@ -373,7 +386,7 @@ def main(versions_file, monorepo): if path not in excluded_dependencies_list \ and path not in main_module.artifact_id: required_dependencies[path] = module.Module( - group_id=_proto_group_id(group_id), + group_id=__proto_group_id(group_id), artifact_id=path, version=main_module.version, release_version=main_module.release_version, @@ -386,11 +399,12 @@ def main(versions_file, monorepo): module=required_dependencies[path], parent_module=parent_module, main_module=main_module, + monorepo=monorepo, ) if path not in excluded_dependencies_list \ and path not in main_module.artifact_id: required_dependencies[path] = module.Module( - group_id=_proto_group_id(group_id), + group_id=__proto_group_id(group_id), artifact_id=path, version=main_module.version, release_version=main_module.release_version, @@ -399,7 +413,7 @@ def main(versions_file, monorepo): for path in glob.glob("grpc-google-*"): if not path in existing_modules: existing_modules[path] = module.Module( - group_id=_proto_group_id(group_id), + group_id=__proto_group_id(group_id), artifact_id=path, version=main_module.version, release_version=main_module.release_version, @@ -407,7 +421,7 @@ def main(versions_file, monorepo): if path not in excluded_dependencies_list \ and path not in main_module.artifact_id: required_dependencies[path] = module.Module( - group_id=_proto_group_id(group_id), + group_id=__proto_group_id(group_id), artifact_id=path, version=main_module.version, release_version=main_module.release_version, @@ -423,11 +437,12 @@ def main(versions_file, monorepo): parent_module=parent_module, main_module=main_module, proto_module=existing_modules[proto_artifact_id], + monorepo=monorepo, ) if path not in excluded_dependencies_list \ and path not in main_module.artifact_id: required_dependencies[path] = module.Module( - group_id=_proto_group_id(group_id), + group_id=__proto_group_id(group_id), artifact_id=path, version=main_module.version, release_version=main_module.release_version, @@ -469,6 +484,7 @@ def main(versions_file, monorepo): description=repo_metadata["api_description"], proto_modules=proto_modules, grpc_modules=grpc_modules, + monorepo=monorepo, ) if os.path.isfile(f"{artifact_id}-bom/pom.xml"): @@ -477,6 +493,8 @@ def main(versions_file, monorepo): update_bom_pom(f"{artifact_id}-bom/pom.xml", modules) elif artifact_id+"-bom" not in excluded_poms_list: print("creating missing bom pom.xml") + monorepo_version = __get_monorepo_version(versions_file) \ + if monorepo else "" templates.render( template_name="bom_pom.xml.j2", output_name=f"{artifact_id}-bom/pom.xml", @@ -484,6 +502,8 @@ def main(versions_file, monorepo): name=name, modules=modules, main_module=main_module, + monorepo=monorepo, + monorepo_version=monorepo_version ) if os.path.isfile("pom.xml"): @@ -491,6 +511,8 @@ def main(versions_file, monorepo): update_parent_pom("pom.xml", modules) else: print("creating missing parent pom.xml") + monorepo_version = __get_monorepo_version(versions_file) \ + if monorepo else "" templates.render( template_name="parent_pom.xml.j2", output_name="./pom.xml", @@ -498,25 +520,24 @@ def main(versions_file, monorepo): modules=modules, main_module=main_module, name=name, + monorepo=monorepo, + monorepo_version=monorepo_version ) - # For monorepo, we use the versions.txt at the root. The "./" is needed - # for the templates.render(), which tries to create a directory. - versions_txt_file = "../versions.txt" if monorepo else "./versions.txt" - print(f"updating modules in {versions_txt_file}") + print(f"updating modules in {versions_file}") existing_modules.pop(parent_artifact_id) # add extra modules to versions.txt for dependency_module in extra_managed_modules: if dependency_module not in existing_modules: existing_modules[dependency_module] = module.Module( - group_id=_proto_group_id(group_id), + group_id=__proto_group_id(group_id), artifact_id=dependency_module, version=main_module.version, release_version=main_module.release_version, ) templates.render( - template_name="versions.txt.j2", output_name=versions_txt_file, modules=existing_modules.values(), + template_name="versions.txt.j2", output_name=versions_file, modules=existing_modules.values(), ) @@ -524,5 +545,5 @@ def main(versions_file, monorepo): versions_file = sys.argv[1] monorepo = sys.argv[2] if monorepo == 'true': - monorepo = True + monorepo = True main(versions_file, monorepo) diff --git a/library_generation/owlbot/templates/poms/bom_pom.xml.j2 b/library_generation/owlbot/templates/poms/bom_pom.xml.j2 index 45e6d25253..ddcef5226f 100644 --- a/library_generation/owlbot/templates/poms/bom_pom.xml.j2 +++ b/library_generation/owlbot/templates/poms/bom_pom.xml.j2 @@ -1,22 +1,30 @@ - + 4.0.0 {{main_module.group_id}} {{main_module.artifact_id}}-bom {{main_module.version}} pom + {% if monorepo -%} + + com.google.cloud + google-cloud-pom-parent + {{ monorepo_version }} + ../../google-cloud-pom-parent/pom.xml + + {%- else -%} com.google.cloud google-cloud-shared-config 1.5.3 + {%- endif %} Google {{name}} BOM BOM for {{name}} - true @@ -30,5 +38,4 @@ {% endfor %} - diff --git a/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 b/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 index 6f999f4897..ed1f6fbfea 100644 --- a/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 +++ b/library_generation/owlbot/templates/poms/cloud_pom.xml.j2 @@ -1,4 +1,4 @@ - + 4.0.0 {{module.group_id}} @@ -6,7 +6,9 @@ {{module.version}} jar Google {{name}} + {%- if not monorepo %} https://github.com/{{repo}} + {%- endif %} {{name}} {{description}} {{parent_module.group_id}} @@ -62,6 +64,10 @@ com.google.api gax-httpjson + + com.google.api.grpc + grpc-google-common-protos + com.google.api.grpc proto-google-iam-v1 @@ -108,6 +114,7 @@ + {%- if not monorepo %} java9 @@ -131,4 +138,5 @@ + {%- endif %} diff --git a/library_generation/owlbot/templates/poms/grpc_pom.xml.j2 b/library_generation/owlbot/templates/poms/grpc_pom.xml.j2 index ad2b39c223..514861e7a7 100644 --- a/library_generation/owlbot/templates/poms/grpc_pom.xml.j2 +++ b/library_generation/owlbot/templates/poms/grpc_pom.xml.j2 @@ -43,6 +43,7 @@ + {%- if not monorepo %} java9 @@ -66,4 +67,5 @@ + {%- endif %} diff --git a/library_generation/owlbot/templates/poms/parent_pom.xml.j2 b/library_generation/owlbot/templates/poms/parent_pom.xml.j2 index cbe3f10913..dcf922340e 100644 --- a/library_generation/owlbot/templates/poms/parent_pom.xml.j2 +++ b/library_generation/owlbot/templates/poms/parent_pom.xml.j2 @@ -10,11 +10,20 @@ Java idiomatic client for Google Cloud Platform services. + {% if monorepo -%} + + com.google.cloud + google-cloud-jar-parent + {{ monorepo_version }} + ../google-cloud-jar-parent/pom.xml + + {%- else -%} com.google.cloud google-cloud-shared-config 1.5.3 + {%- endif %} UTF-8 @@ -34,7 +43,6 @@ - {% for module in modules %} {{module.artifact_id}} {% endfor %} {{main_module.artifact_id}}-bom diff --git a/library_generation/owlbot/templates/poms/proto_pom.xml.j2 b/library_generation/owlbot/templates/poms/proto_pom.xml.j2 index 9c383533c7..886cd02663 100644 --- a/library_generation/owlbot/templates/poms/proto_pom.xml.j2 +++ b/library_generation/owlbot/templates/poms/proto_pom.xml.j2 @@ -35,6 +35,7 @@ + {%- if not monorepo %} @@ -43,4 +44,5 @@ + {%- endif %} diff --git a/library_generation/postprocess_library.sh b/library_generation/postprocess_library.sh index bb8415dda2..bf07127427 100755 --- a/library_generation/postprocess_library.sh +++ b/library_generation/postprocess_library.sh @@ -88,16 +88,6 @@ docker run --rm \ --source-repo=/pre-processed-libraries \ --config-file=.OwlBot.yaml -# if the postprocessing_target is a library of google-cloud-java, we have to "unpack" the -# owl-bot-staging folder so it's properly processed by java owlbot -if [[ $(basename $(dirname "${postprocessing_target}")) == "google-cloud-java" ]]; then - pushd "${postprocessing_target}" - mv owl-bot-staging/* temp - rm -rd owl-bot-staging/ - mv temp owl-bot-staging - popd # postprocessing_target -fi - # we clone the synthtool library and manually build it mkdir -p /tmp/synthtool pushd /tmp/synthtool diff --git a/library_generation/repo-level-postprocess/generate_gapic_bom.sh b/library_generation/repo-level-postprocess/generate_gapic_bom.sh new file mode 100755 index 0000000000..ad37553d58 --- /dev/null +++ b/library_generation/repo-level-postprocess/generate_gapic_bom.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +set -e + +# Generate BOM of the artifacts in this repository +GENERATION_DIR=$1 +bom_lines="" +# For modules that produce BOMs +for bom_directory in $(find . -maxdepth 3 -name 'google-*-bom' | sort --dictionary-order); do + if [[ "${bom_directory}" = *gapic-libraries-bom ]] || [[ "${bom_directory}" = *google-cloud-core* ]]; then + continue + fi + pom_file="${bom_directory}/pom.xml" + groupId_line=$(grep --max-count=1 'groupId' "${pom_file}") + artifactId_line=$(grep --max-count=1 'artifactId' "${pom_file}") + version_line=$(grep --max-count=1 'x-version-update' "${pom_file}") + + if [[ "$groupId_line" == *"com.google.cloud"* + || "$groupId_line" == *"com.google.analytic"* + || "$groupId_line" == *"com.google.area120"* + || "$groupId_line" == *"io.grafeas"* ]]; then + # The gapic bom mainly includes cloud libraries and ones that have been included already. + # Let's avoid adding com.google.maps and com.google.shopping for now. We may decide to + # add them later. It's more difficult to remove them later without impacting users. + bom_lines+=" \n\ + ${groupId_line}\n\ + ${artifactId_line}\n\ + ${version_line}\n\ + pom\n\ + import\n\ + \n" + fi +done + +# For originally-handwritten modules that do not produce a BOM +for module in $(find . -mindepth 2 -maxdepth 2 -name pom.xml |sort --dictionary-order | xargs dirname); do + if ls "${module}"/*-bom 1> /dev/null 2>&1; then + continue + fi + if ! test -f "${module}/.repo-metadata.json"; then + continue + fi + + pom_file="${module}/pom.xml" + groupId_line=$(grep --max-count=1 'groupId' "${pom_file}") + artifactId_line=$(grep --max-count=1 'artifactId' "${pom_file}") + version_line=$(grep --max-count=1 'x-version-update' "${pom_file}") + bom_lines+=" \n\ + ${groupId_line}\n\ + ${artifactId_line}\n\ + ${version_line}\n\ + \n" +done + +mkdir -p gapic-libraries-bom + +perl -0pe 's/.*<\/dependencies>/\nBOM_ARTIFACT_LIST\n <\/dependencies>/s' "${GENERATION_DIR}/../gapic-libraries-bom/pom.xml" > "${GENERATION_DIR}/bom.pom.xml" +awk -v "dependencyManagements=${bom_lines}" '{gsub(/BOM_ARTIFACT_LIST/,dependencyManagements)}1' \ + "${GENERATION_DIR}/bom.pom.xml" > gapic-libraries-bom/pom.xml +rm "${GENERATION_DIR}/bom.pom.xml" \ No newline at end of file diff --git a/library_generation/repo-level-postprocess/generate_root_pom.sh b/library_generation/repo-level-postprocess/generate_root_pom.sh new file mode 100755 index 0000000000..2cac682c94 --- /dev/null +++ b/library_generation/repo-level-postprocess/generate_root_pom.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -e + +GENERATION_DIR=$1; + +# Find all Maven modules (a directory that contains pom.xml) +find . -mindepth 2 -maxdepth 2 -name pom.xml |sort --dictionary-order | xargs dirname \ + |sed -e 's|./||' | xargs -I '{}' echo " {}" > /tmp/repo-modules.txt + +perl -0pe 's/.*<\/modules>/\n <\/modules>/s' ${GENERATION_DIR}/../pom.xml > ${GENERATION_DIR}/parent.pom.xml +awk -v MODULES="`awk -v ORS='\\\\n' '1' /tmp/repo-modules.txt`" '1;//{print MODULES}' ${GENERATION_DIR}/parent.pom.xml > pom.xml +rm ${GENERATION_DIR}/parent.pom.xml \ No newline at end of file