diff --git a/.gitignore b/.gitignore index c8f0fb448f78..08c8434fbbf1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,34 @@ -# Packages +# Python Packages +# Exerpt from GitHub's recommendation +# https://github.com/github/gitignore/blob/main/Python.gitignore +.Python dist bin var sdist target +eggs/ +.eggs/ +sdist/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Python utilities +__pycache__/ +*.py[cod] +*$py.class + +# Python environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + # Unit test / coverage reports .coverage @@ -13,6 +38,7 @@ nosetests.xml # Translations *.mo +# Irrelevant files generated by IDEs and OSes .metadata .project .pydevproject @@ -28,7 +54,9 @@ docs/ # API key file containing value of GOOGLE_API_KEY for integration tests api_key -# Python utilities -*.pyc +# New client generation +generation/new_client/workspace + +# Monorepo repository generation +monorepo/ -monorepo/ \ No newline at end of file diff --git a/generation/new_client/README.md b/generation/new_client/README.md new file mode 100644 index 000000000000..febeba50decc --- /dev/null +++ b/generation/new_client/README.md @@ -0,0 +1,174 @@ +# New Google Client Library Generation + +The script allows you to create a new client library module in the +google-cloud-java monorepo. + +**This tool is for repository maintainers only. Not for library users.** + +## Prerequisites + +This section is only needed for the first run of this script. If it's already +done, go to "Run client generation script" section. + + +### Environment + +Use Linux environment. + +Install Docker. + +### Checkout google-cloud-java repository + +``` +$ git clone https://github.com/googleapis/google-cloud-java +``` + +### Install pyenv + +Install pyenv + +``` +curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer \ +| bash +``` + +Follow the instruction in the output above to append lines in `$HOME/.bashrc`. +Logout the shell and login again. + +Confirm pyenv installation succeeded: + +``` +suztomo@suztomo:~$ pyenv +pyenv 2.3.4 +Usage: pyenv [] + +Some useful pyenv commands are: + activate Activate virtual environment + commands List all available pyenv commands + deactivate Deactivate virtual environment +... +``` + +### Install Python 3.9 via pyenv + +``` +suztomo@suztomo:~$ pyenv install 3.9.13 +Downloading Python-3.9.13.tar.xz... +-> https://www.python.org/ftp/python/3.9.13/Python-3.9.13.tar.xz +Installing Python-3.9.13... +WARNING: The Python sqlite3 extension was not compiled. Missing the SQLite3 lib? +Installed Python-3.9.13 to /usr/local/google/home/suztomo/.pyenv/versions/3.9.13 +``` + +Conform `python3.9` command is available: + +``` +$ pyenv local 3.9.13 +$ which python3.9 +/usr/local/google/home/suztomo/.pyenv/shims/python3.9 +``` + +### Install Python packages + +At the root of google-cloud-java repository clone, run: + +``` +$ python3.9 -m pip install -r generation/new_client/requirements.txt +``` + +## Run client generation script + +You will run new-client.py script with the following parameters. +Collect them from the ticket before running the command. + +### API short name + +For convenience of the subsequent commands, define a variable for API short name. +Get the value from the DevRel Services page (Example: `apikeys`): + +### proto path + +The script takes "proto path" parameter. This is path from google3's root to the +directory that contains versions (e.g., "v1" or "v2"). For example, if we +have + +> Link to protos: http://google3/google/api/apikeys/v2/apikeys.proto + +then the "proto path" value we supply to the command is "google/api/apikeys". + +### Name pretty + +Pick name from the display name in the DevRel Services site. +For example, "API Keys API" in +https://devrel.corp.google.com/admin/products/479. + +### Product Docs + +Find product document URL in the DevRel Services site. +For example, "https://cloud.google.com/api-keys/" in +https://devrel.corp.google.com/admin/products/479. + +### API description + +Find the short description of the API. Usually the first sentence in the product +document above. + +### Release level + +The first client generation is always "preview". + +### Transport + +Transport layer. Use "grpc". + + +### Language + +Use "java" + +### Monorepo URL + +Use "https://github.com/googleapis/google-cloud-java" + + +### Example arguments + +Run `new-client.py` with the arguments above: + +``` +$ cd generation/new_client +$ python3.9 new-client.py generate \ + --api_shortname=apikeys \ + --proto-path=google/api/apikeys \ + --name-pretty="API Keys API" \ + --product-docs="https://cloud.google.com/api-keys/" \ + --api-description="API Keys lets you create and manage your API keys for your projects." \ + --release-level=preview \ + --transport=grpc \ + --language=java \ + --monorepo-url="https://github.com/googleapis/google-cloud-java" +``` + +The command creates `workspace` directory in which it prepares the changes for +the new module in the monorepo. At the end (~ 10 minutes), it tells you to +create a pull request in the monorepo: + +``` +... +Prepared new library in workspace/monorepo/java-apikeys +Please create a pull request from that directory: + $ cd /usr/local/google/home/suztomo/google-cloud-java/generation/new_client/workspace/monorepo + $ gh pr create --title 'feat: [apikeys] new module for apikeys' +``` + +Create a pull request from the change. + +# Principles + +The script should finish creating a pull request even when the newly created +module fails to compile. This gives the user flexibility to fix things in the +created pull request. + + + + diff --git a/generation/new_client/new-client.py b/generation/new_client/new-client.py new file mode 100644 index 000000000000..a9a46e9aedc0 --- /dev/null +++ b/generation/new_client/new-client.py @@ -0,0 +1,358 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +from pathlib import Path +import re +import subprocess +import sys + +import click +import templates + + +@click.group(invoke_without_command=False) +@click.pass_context +@click.version_option(message="%(version)s") +def main(ctx): + pass + +def add_module_to_root_pom(pom_path: Path, new_module: str): + with open(pom_path, "r") as fp: + content = fp.read() + + matches = re.findall(r'([\w\-]+?)', content) + matches.append(new_module) + + # Make normal modules first and BOM and CoverageAggregator at the end + matches.sort(key=lambda m: f"0{m}" if m.startswith('java-') else m) + modules_lines = [] + for match in matches: + modules_lines.append(f"{match}") + + modules_line = "\n ".join(modules_lines) + ordered_content = re.sub(r".+", + f"\n {modules_line}\n ", + content, + flags=re.MULTILINE | re.DOTALL) + + with open(pom_path, "w") as fp: + fp.truncate(0) + fp.write(ordered_content) + +@main.command() +@click.option("--api_shortname", required=True, type=str, prompt="Service name? (e.g. automl)") +@click.option( + "--name-pretty", + required=True, + type=str, + prompt="Pretty name? (e.g. 'Cloud AutoML')", +) +@click.option( + "--product-docs", required=True, type=str, prompt="Product Documentation URL" +) +@click.option( + "--api-description", + required=True, + type=str, + prompt="Description for README. The first sentence is prefixed by the pretty name", +) +@click.option( + "--release-level", + required=True, + type=click.Choice(["stable", "preview"]), + prompt=True, + default="preview", +) +@click.option( + "--transport", + required=True, + type=click.Choice(["grpc", "http", "both"]), + prompt=True, + default="grpc", +) +@click.option("--language", required=True, type=str, default="java") +@click.option("--distribution-name", type=str) +@click.option("--api-id", type=str) +@click.option("--requires-billing", type=bool, default=True) +@click.option("--destination-name", type=str, default=None) +@click.option("--proto-path", type=str, default=None) +@click.option("--cloud-api", type=bool, default=True) +@click.option("--group-id", type=str, default="com.google.cloud") +@click.option( + "--owlbot-image", type=str, default="gcr.io/cloud-devrel-public-resources/owlbot-java" +) +@click.option("--library-type", type=str) +@click.option("--monorepo-url", type=str, default=None) +def generate( + api_shortname, + name_pretty, + product_docs, + api_description, + release_level, + distribution_name, + api_id, + requires_billing, + transport, + language, + destination_name, + proto_path, + cloud_api, + group_id, + owlbot_image, + library_type, + monorepo_url, +): + cloud_prefix = "cloud-" if cloud_api else "" + + output_name = destination_name if destination_name else api_shortname + if distribution_name is None: + distribution_name = f"{group_id}:google-{cloud_prefix}{output_name}" + + distribution_name_short = re.split(r"[:\/]", distribution_name)[-1] + + if api_id is None: + api_id = f"{api_shortname}.googleapis.com" + + if library_type is None: + library_type = "GAPIC_AUTO" + + client_documentation = ( + f"https://cloud.google.com/{language}/docs/reference/{distribution_name_short}/latest/overview" + ) + + if destination_name is None: + destination_name = api_shortname + + if proto_path is None: + proto_path = f"/google/cloud/{api_shortname}" + + if api_shortname == "": + sys.exit("api_shortname is empty") + + repo_metadata = { + "api_shortname": api_shortname, + "name_pretty": name_pretty, + "product_documentation": product_docs, + "api_description": api_description, + "client_documentation": client_documentation, + "release_level": release_level, + "transport": transport, + "language": language, + "repo": f"googleapis/{language}-{output_name}", + "repo_short": f"{language}-{output_name}", + "distribution_name": distribution_name, + "api_id": api_id, + "library_type": library_type, + } + if requires_billing: + repo_metadata["requires_billing"] = True + + # Initialize workdir + print("Creating a new module in monorepo " + monorepo_url) + subprocess.check_call(["mkdir", "-p", "workspace"]) + subprocess.check_call(["rm", "-fr", "monorepo"], cwd="workspace") + subprocess.check_call(["git", "clone", monorepo_url, "monorepo"], + cwd="workspace") + workdir = Path(f"workspace/monorepo/java-{output_name}") + os.makedirs(workdir, exist_ok=True) + subprocess.check_call(["git", "checkout", "-b", + f"new_module_java-{output_name}"], + cwd=workdir) + add_module_to_root_pom(workdir / ".." / "pom.xml", + f"java-{output_name}") + subprocess.check_call(["git", "add", "."], cwd=workdir / "..") + # write .repo-metadata.json file + with open(workdir / ".repo-metadata.json", "w") as fp: + json.dump(repo_metadata, fp, indent=2) + + # create owlbot.py + templates.render( + template_name="owlbot.py.j2", + output_name=str(workdir / "owlbot.py"), + should_include_templates=True, + template_excludes=[], + ) + + # In monorepo, .OwlBot.yaml needs to be in the directory of the module. + owlbot_yaml_location_from_module = ".OwlBot.yaml" + # create owlbot config + templates.render( + template_name="owlbot.yaml.monorepo.j2", + output_name=str(workdir / owlbot_yaml_location_from_module), + artifact_name=distribution_name_short, + proto_path=proto_path, + module_name=f"java-{output_name}" + ) + + # get the sha256 digets for the owlbot image + subprocess.check_call(["docker", "pull", owlbot_image]) + owlbot_image_digest = ( + subprocess.check_output( + ["docker", "inspect", "--format='{{index .RepoDigests 0}}", owlbot_image,], + encoding="utf-8", + ) + .strip() + .split("@")[-1] + ) + + user = subprocess.check_output(["id", "-u"], encoding="utf8").strip() + group = subprocess.check_output(["id", "-g"], encoding="utf8").strip() + + # run owlbot copy + print("Cloning googleapis-gen...") + subprocess.check_call(["git", "clone", "https://github.com/googleapis/googleapis-gen.git", "./gen/googleapis-gen"], cwd=workdir) + subprocess.check_call(["docker", "pull", "gcr.io/cloud-devrel-public-resources/owlbot-cli:latest"]) + print("Running copy-code...") + subprocess.check_call( + [ + "docker", + "run", + "--rm", + "--user", + f"{user}:{group}", + "-v", + f"{workdir.resolve()}:/repo", + "-v", + ""f"{workdir.resolve()}""/gen/googleapis-gen:/googleapis-gen", + "-w", + "/repo", + "--env", "HOME=/tmp", + "gcr.io/cloud-devrel-public-resources/owlbot-cli:latest", + "copy-code", + "--source-repo=/googleapis-gen", + f"--config-file={owlbot_yaml_location_from_module}" + ], + cwd=workdir, + ) + + print("Removing googleapis-gen...") + subprocess.check_call(["rm", "-fr", "gen"], cwd=workdir) + + # Bringing owl-bot-staging from the new module's directory to the root + # directory so that owlbot-java can process them. + subprocess.check_call( + [ + "mv", + "owl-bot-staging", + "../" + ], + cwd=workdir, + ) + monorepo_root=(workdir / '..').resolve() + print("Running the post-processor...") + subprocess.check_call( + [ + "docker", + "run", + "--rm", + "-v", + f"{monorepo_root}:/workspace", + "--user", + f"{user}:{group}", + owlbot_image, + ], + cwd=monorepo_root, + ) + # In monorpeo, .github and .kokoro under the module is unused + print("Removing unnecessary files") + subprocess.check_call(["rm", "-fr", ".github"], + cwd=workdir) + subprocess.check_call(["rm", "-fr", ".kokoro"], + cwd=workdir) + subprocess.check_call(["rm", "-f", ".gitignore"], + cwd=workdir) + + # Remove irrelevant files from templates + subprocess.check_call( + ["bash", "generation/update_owlbot_postprocessor_config.sh"], + cwd=monorepo_root + ) + subprocess.check_call( + ["bash", "generation/delete_non_generated_samples.sh"], + cwd=monorepo_root + ) + + subprocess.check_call(["git", "add", "."], cwd=workdir) + subprocess.check_call( + ["git", "commit", "-m", f"feat: initial generation of {api_shortname}"], cwd=workdir + ) + + print("Regenerating the BOM") + subprocess.check_call( + [ + "bash", "generation/generate_gapic_bom.sh", + ], + cwd=monorepo_root, + ) + + print("Regenerating CoverageAggregator module and root pom.xml") + + # This script takes care of updating the root pom.xml + os.system(f"cd {monorepo_root} && generation/print_root_pom.sh > pom.xml") + + # This script updates every module's pom sets the root as parent + subprocess.check_call( + [ + "bash", "generation/set_parent_pom.sh" + ], + cwd=monorepo_root, + ) + + print("Regenerating Release Please config files") + subprocess.check_call( + [ + "bash", "generation/generate_release_please_config.sh" + ], + cwd=monorepo_root, + ) + print("Regenerating CoverageAggregator") + subprocess.check_call( + [ + "bash", "generation/generate_coverage_aggregator.sh" + ], + cwd=monorepo_root, + ) + + # Add the files to commit + subprocess.check_call([ + "git", "add", "pom.xml", "google-cloud-gapic-bom/pom.xml", + "release-please-config.json", ".release-please-manifest.json", + "CoverageAggregator/pom.xml"], + cwd=monorepo_root) + + subprocess.check_call( + ["git", "commit", "-m", + f"build: add the {api_shortname} module to monorepo"], + cwd=monorepo_root + ) + + # Remove irrelevant changes in other modules + subprocess.check_call( + ["git", "checkout", "--", "."], + cwd=monorepo_root + ) + + # It seems generate_release_please_config.sh is not ready to run as + # part of client library generation process. + + print(f"Prepared new library in {workdir}") + print(f"Please create a pull request from that directory:\n" + f" $ cd {monorepo_root}\n" + f" $ gh pr create --title 'feat: [{api_shortname}] new module for {api_shortname}'") + +if __name__ == "__main__": + main() + diff --git a/generation/new_client/requirements.in b/generation/new_client/requirements.in new file mode 100644 index 000000000000..81fa45879151 --- /dev/null +++ b/generation/new_client/requirements.in @@ -0,0 +1,7 @@ +attr +attrs +black +click==8.0 +jinja2 +lxml==4.4.1 +typing==3.7.4.1 diff --git a/generation/new_client/requirements.txt b/generation/new_client/requirements.txt new file mode 100644 index 000000000000..96195384ded7 --- /dev/null +++ b/generation/new_client/requirements.txt @@ -0,0 +1,140 @@ +# +# This file is autogenerated by pip-compile with python 3.10 +# To update, run: +# +# pip-compile --generate-hashes requirements.in +# +attr==0.3.2 \ + --hash=sha256:1ceebca768181cdcce9827611b1d728e592be5d293911539ea3d0b0bfa1146f4 \ + --hash=sha256:4f4bffeea8c27387bde446675a7ac24f3b8fea1075f12d849b5f5c5181fc8336 + # via -r requirements.in +attrs==22.1.0 \ + --hash=sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6 \ + --hash=sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c + # via -r requirements.in +black==22.8.0 \ + --hash=sha256:0a12e4e1353819af41df998b02c6742643cfef58282915f781d0e4dd7a200411 \ + --hash=sha256:0ad827325a3a634bae88ae7747db1a395d5ee02cf05d9aa7a9bd77dfb10e940c \ + --hash=sha256:32a4b17f644fc288c6ee2bafdf5e3b045f4eff84693ac069d87b1a347d861497 \ + --hash=sha256:3b2c25f8dea5e8444bdc6788a2f543e1fb01494e144480bc17f806178378005e \ + --hash=sha256:4a098a69a02596e1f2a58a2a1c8d5a05d5a74461af552b371e82f9fa4ada8342 \ + --hash=sha256:5107ea36b2b61917956d018bd25129baf9ad1125e39324a9b18248d362156a27 \ + --hash=sha256:53198e28a1fb865e9fe97f88220da2e44df6da82b18833b588b1883b16bb5d41 \ + --hash=sha256:5594efbdc35426e35a7defa1ea1a1cb97c7dbd34c0e49af7fb593a36bd45edab \ + --hash=sha256:5b879eb439094751185d1cfdca43023bc6786bd3c60372462b6f051efa6281a5 \ + --hash=sha256:78dd85caaab7c3153054756b9fe8c611efa63d9e7aecfa33e533060cb14b6d16 \ + --hash=sha256:792f7eb540ba9a17e8656538701d3eb1afcb134e3b45b71f20b25c77a8db7e6e \ + --hash=sha256:8ce13ffed7e66dda0da3e0b2eb1bdfc83f5812f66e09aca2b0978593ed636b6c \ + --hash=sha256:a05da0430bd5ced89176db098567973be52ce175a55677436a271102d7eaa3fe \ + --hash=sha256:a983526af1bea1e4cf6768e649990f28ee4f4137266921c2c3cee8116ae42ec3 \ + --hash=sha256:bc4d4123830a2d190e9cc42a2e43570f82ace35c3aeb26a512a2102bce5af7ec \ + --hash=sha256:c3a73f66b6d5ba7288cd5d6dad9b4c9b43f4e8a4b789a94bf5abfb878c663eb3 \ + --hash=sha256:ce957f1d6b78a8a231b18e0dd2d94a33d2ba738cd88a7fe64f53f659eea49fdd \ + --hash=sha256:cea1b2542d4e2c02c332e83150e41e3ca80dc0fb8de20df3c5e98e242156222c \ + --hash=sha256:d2c21d439b2baf7aa80d6dd4e3659259be64c6f49dfd0f32091063db0e006db4 \ + --hash=sha256:d839150f61d09e7217f52917259831fe2b689f5c8e5e32611736351b89bb2a90 \ + --hash=sha256:dd82842bb272297503cbec1a2600b6bfb338dae017186f8f215c8958f8acf869 \ + --hash=sha256:e8166b7bfe5dcb56d325385bd1d1e0f635f24aae14b3ae437102dedc0c186747 \ + --hash=sha256:e981e20ec152dfb3e77418fb616077937378b322d7b26aa1ff87717fb18b4875 + # via -r requirements.in +click==8.0 \ + --hash=sha256:7d8c289ee437bcb0316820ccee14aefcb056e58d31830ecab8e47eda6540e136 \ + --hash=sha256:e90e62ced43dc8105fb9a26d62f0d9340b5c8db053a814e25d95c19873ae87db + # via + # -r requirements.in + # black +jinja2==3.1.2 \ + --hash=sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 \ + --hash=sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61 + # via -r requirements.in +lxml==4.4.1 \ + --hash=sha256:02ca7bf899da57084041bb0f6095333e4d239948ad3169443f454add9f4e9cb4 \ + --hash=sha256:096b82c5e0ea27ce9138bcbb205313343ee66a6e132f25c5ed67e2c8d960a1bc \ + --hash=sha256:0a920ff98cf1aac310470c644bc23b326402d3ef667ddafecb024e1713d485f1 \ + --hash=sha256:1409b14bf83a7d729f92e2a7fbfe7ec929d4883ca071b06e95c539ceedb6497c \ + --hash=sha256:17cae1730a782858a6e2758fd20dd0ef7567916c47757b694a06ffafdec20046 \ + --hash=sha256:17e3950add54c882e032527795c625929613adbd2ce5162b94667334458b5a36 \ + --hash=sha256:1f4f214337f6ee5825bf90a65d04d70aab05526c08191ab888cb5149501923c5 \ + --hash=sha256:2e8f77db25b0a96af679e64ff9bf9dddb27d379c9900c3272f3041c4d1327c9d \ + --hash=sha256:4dffd405390a45ecb95ab5ab1c1b847553c18b0ef8ed01e10c1c8b1a76452916 \ + --hash=sha256:6b899931a5648862c7b88c795eddff7588fb585e81cecce20f8d9da16eff96e0 \ + --hash=sha256:726c17f3e0d7a7200718c9a890ccfeab391c9133e363a577a44717c85c71db27 \ + --hash=sha256:760c12276fee05c36f95f8040180abc7fbebb9e5011447a97cdc289b5d6ab6fc \ + --hash=sha256:796685d3969815a633827c818863ee199440696b0961e200b011d79b9394bbe7 \ + --hash=sha256:891fe897b49abb7db470c55664b198b1095e4943b9f82b7dcab317a19116cd38 \ + --hash=sha256:9277562f175d2334744ad297568677056861070399cec56ff06abbe2564d1232 \ + --hash=sha256:a471628e20f03dcdfde00770eeaf9c77811f0c331c8805219ca7b87ac17576c5 \ + --hash=sha256:a63b4fd3e2cabdcc9d918ed280bdde3e8e9641e04f3c59a2a3109644a07b9832 \ + --hash=sha256:ae88588d687bd476be588010cbbe551e9c2872b816f2da8f01f6f1fda74e1ef0 \ + --hash=sha256:b0b84408d4eabc6de9dd1e1e0bc63e7731e890c0b378a62443e5741cfd0ae90a \ + --hash=sha256:be78485e5d5f3684e875dab60f40cddace2f5b2a8f7fede412358ab3214c3a6f \ + --hash=sha256:c27eaed872185f047bb7f7da2d21a7d8913457678c9a100a50db6da890bc28b9 \ + --hash=sha256:c7fccd08b14aa437fe096c71c645c0f9be0655a9b1a4b7cffc77bcb23b3d61d2 \ + --hash=sha256:c81cb40bff373ab7a7446d6bbca0190bccc5be3448b47b51d729e37799bb5692 \ + --hash=sha256:d11874b3c33ee441059464711cd365b89fa1a9cf19ae75b0c189b01fbf735b84 \ + --hash=sha256:e9c028b5897901361d81a4718d1db217b716424a0283afe9d6735fe0caf70f79 \ + --hash=sha256:fe489d486cd00b739be826e8c1be188ddb74c7a1ca784d93d06fda882a6a1681 + # via -r requirements.in +markupsafe==2.1.1 \ + --hash=sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003 \ + --hash=sha256:089cf3dbf0cd6c100f02945abeb18484bd1ee57a079aefd52cffd17fba910b88 \ + --hash=sha256:10c1bfff05d95783da83491be968e8fe789263689c02724e0c691933c52994f5 \ + --hash=sha256:33b74d289bd2f5e527beadcaa3f401e0df0a89927c1559c8566c066fa4248ab7 \ + --hash=sha256:3799351e2336dc91ea70b034983ee71cf2f9533cdff7c14c90ea126bfd95d65a \ + --hash=sha256:3ce11ee3f23f79dbd06fb3d63e2f6af7b12db1d46932fe7bd8afa259a5996603 \ + --hash=sha256:421be9fbf0ffe9ffd7a378aafebbf6f4602d564d34be190fc19a193232fd12b1 \ + --hash=sha256:43093fb83d8343aac0b1baa75516da6092f58f41200907ef92448ecab8825135 \ + --hash=sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247 \ + --hash=sha256:4a33dea2b688b3190ee12bd7cfa29d39c9ed176bda40bfa11099a3ce5d3a7ac6 \ + --hash=sha256:4b9fe39a2ccc108a4accc2676e77da025ce383c108593d65cc909add5c3bd601 \ + --hash=sha256:56442863ed2b06d19c37f94d999035e15ee982988920e12a5b4ba29b62ad1f77 \ + --hash=sha256:671cd1187ed5e62818414afe79ed29da836dde67166a9fac6d435873c44fdd02 \ + --hash=sha256:694deca8d702d5db21ec83983ce0bb4b26a578e71fbdbd4fdcd387daa90e4d5e \ + --hash=sha256:6a074d34ee7a5ce3effbc526b7083ec9731bb3cbf921bbe1d3005d4d2bdb3a63 \ + --hash=sha256:6d0072fea50feec76a4c418096652f2c3238eaa014b2f94aeb1d56a66b41403f \ + --hash=sha256:6fbf47b5d3728c6aea2abb0589b5d30459e369baa772e0f37a0320185e87c980 \ + --hash=sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b \ + --hash=sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812 \ + --hash=sha256:8dc1c72a69aa7e082593c4a203dcf94ddb74bb5c8a731e4e1eb68d031e8498ff \ + --hash=sha256:8e3dcf21f367459434c18e71b2a9532d96547aef8a871872a5bd69a715c15f96 \ + --hash=sha256:8e576a51ad59e4bfaac456023a78f6b5e6e7651dcd383bcc3e18d06f9b55d6d1 \ + --hash=sha256:96e37a3dc86e80bf81758c152fe66dbf60ed5eca3d26305edf01892257049925 \ + --hash=sha256:97a68e6ada378df82bc9f16b800ab77cbf4b2fada0081794318520138c088e4a \ + --hash=sha256:99a2a507ed3ac881b975a2976d59f38c19386d128e7a9a18b7df6fff1fd4c1d6 \ + --hash=sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e \ + --hash=sha256:b09bf97215625a311f669476f44b8b318b075847b49316d3e28c08e41a7a573f \ + --hash=sha256:b7bd98b796e2b6553da7225aeb61f447f80a1ca64f41d83612e6139ca5213aa4 \ + --hash=sha256:b87db4360013327109564f0e591bd2a3b318547bcef31b468a92ee504d07ae4f \ + --hash=sha256:bcb3ed405ed3222f9904899563d6fc492ff75cce56cba05e32eff40e6acbeaa3 \ + --hash=sha256:d4306c36ca495956b6d568d276ac11fdd9c30a36f1b6eb928070dc5360b22e1c \ + --hash=sha256:d5ee4f386140395a2c818d149221149c54849dfcfcb9f1debfe07a8b8bd63f9a \ + --hash=sha256:dda30ba7e87fbbb7eab1ec9f58678558fd9a6b8b853530e176eabd064da81417 \ + --hash=sha256:e04e26803c9c3851c931eac40c695602c6295b8d432cbe78609649ad9bd2da8a \ + --hash=sha256:e1c0b87e09fa55a220f058d1d49d3fb8df88fbfab58558f1198e08c1e1de842a \ + --hash=sha256:e72591e9ecd94d7feb70c1cbd7be7b3ebea3f548870aa91e2732960fa4d57a37 \ + --hash=sha256:e8c843bbcda3a2f1e3c2ab25913c80a3c5376cd00c6e8c4a86a89a28c8dc5452 \ + --hash=sha256:efc1913fd2ca4f334418481c7e595c00aad186563bbc1ec76067848c7ca0a933 \ + --hash=sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a \ + --hash=sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7 + # via jinja2 +mypy-extensions==0.4.3 \ + --hash=sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d \ + --hash=sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8 + # via black +pathspec==0.10.1 \ + --hash=sha256:46846318467efc4556ccfd27816e004270a9eeeeb4d062ce5e6fc7a87c573f93 \ + --hash=sha256:7ace6161b621d31e7902eb6b5ae148d12cfd23f4a249b9ffb6b9fee12084323d + # via black +platformdirs==2.5.2 \ + --hash=sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788 \ + --hash=sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19 + # via black +tomli==2.0.1 \ + --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \ + --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f + # via black +typing==3.7.4.1 \ + --hash=sha256:91dfe6f3f706ee8cc32d38edbbf304e9b7583fb37108fef38229617f8b3eba23 \ + --hash=sha256:c8cabb5ab8945cd2f54917be357d134db9cc1eb039e59d1606dc1e60cb1d9d36 \ + --hash=sha256:f38d83c5a7a7086543a0f649564d661859c5146a85775ab90c0d2f93ffaa9714 + # via -r requirements.in diff --git a/generation/new_client/templates.py b/generation/new_client/templates.py new file mode 100644 index 000000000000..3b3af05b6598 --- /dev/null +++ b/generation/new_client/templates.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from jinja2 import Environment, FileSystemLoader +import os +import pathlib +from typing import List + +root_directory = pathlib.Path( + os.path.realpath(os.path.dirname(os.path.realpath(__file__))) +) +print(root_directory) +jinja_env = Environment(loader=FileSystemLoader(str(root_directory / "templates"))) + + +def render(template_name: str, output_name: str, **kwargs): + template = jinja_env.get_template(template_name) + t = template.stream(kwargs) + directory = os.path.dirname(output_name) + if not os.path.isdir(directory): + os.makedirs(directory) + t.dump(str(output_name)) diff --git a/generation/new_client/templates/owlbot.py.j2 b/generation/new_client/templates/owlbot.py.j2 new file mode 100644 index 000000000000..70c67d230d68 --- /dev/null +++ b/generation/new_client/templates/owlbot.py.j2 @@ -0,0 +1,24 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import synthtool as s +{% if should_include_templates %}from synthtool.languages import java{% endif %} + + +for library in s.get_staging_dirs(): + # put any special-case replacements here + s.move(library) + +s.remove_staging_dirs() +{% if should_include_templates %}java.common_templates({% if template_excludes %}excludes={{ template_excludes }}{% endif %}){% endif %} diff --git a/generation/new_client/templates/owlbot.yaml.monorepo.j2 b/generation/new_client/templates/owlbot.yaml.monorepo.j2 new file mode 100644 index 000000000000..30b1b1a08296 --- /dev/null +++ b/generation/new_client/templates/owlbot.yaml.monorepo.j2 @@ -0,0 +1,31 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{% if artifact_name %} +deep-remove-regex: +- "/grpc-google-.*/src" +- "/proto-google-.*/src" +- "/google-.*/src" + +deep-preserve-regex: +- "/google-.*/src/test/java/com/google/cloud/.*/v.*/it/IT.*Test.java" + +deep-copy-regex: +- source: "/{{ proto_path }}/(v.*)/.*-java/proto-google-.*/src" + dest: "/owl-bot-staging/{{ module_name }}/$1/proto-{{ artifact_name }}-$1/src" +- source: "/{{ proto_path }}/(v.*)/.*-java/grpc-google-.*/src" + dest: "/owl-bot-staging/{{ module_name }}/$1/grpc-{{ artifact_name }}-$1/src" +- source: "/{{ proto_path }}/(v.*)/.*-java/gapic-google-.*/src" + dest: "/owl-bot-staging/{{ module_name }}/$1/{{ artifact_name }}/src" +{% endif %}