From d5631cd98eae5e806978f1d2ee590915f5b6abe0 Mon Sep 17 00:00:00 2001 From: Jeff Ching Date: Wed, 28 Nov 2018 10:42:05 -0800 Subject: [PATCH 1/9] Add new client python script --- utilities/new_client.py | 297 ++++++++++++++++++++++++++++++ utilities/templates/README.md | 100 ++++++++++ utilities/templates/cloud_pom.xml | 87 +++++++++ utilities/templates/grpc_pom.xml | 31 ++++ utilities/templates/proto_pom.xml | 36 ++++ utilities/templates/synth.py | 35 ++++ 6 files changed, 586 insertions(+) create mode 100644 utilities/new_client.py create mode 100644 utilities/templates/README.md create mode 100644 utilities/templates/cloud_pom.xml create mode 100644 utilities/templates/grpc_pom.xml create mode 100644 utilities/templates/proto_pom.xml create mode 100644 utilities/templates/synth.py diff --git a/utilities/new_client.py b/utilities/new_client.py new file mode 100644 index 000000000000..ee758e199ad2 --- /dev/null +++ b/utilities/new_client.py @@ -0,0 +1,297 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Usage: python new_client.py -v -s -c + +import argparse +import copy +import os +import re +import sys +import subprocess +from jinja2 import Environment, FileSystemLoader +from lxml import etree +from releasetool.commands.start import java as releasetool +from typing import List + +class Context: + service: str = None + api_version: str = None + artman_config: str = None + google_cloud_artifact: str = None + google_cloud_version: releasetool.Version = None + grpc_artifact: str = None + grpc_version: releasetool.Version = None + proto_artifact: str = None + proto_version: releasetool.Version = None + root_directory: str = None + description: str = "FIXME" + name: str = "FIXME" + versions: List[str] = None + + def __init__(self, service: str, api_version: str, artman_config: str): + self.service = service + self.api_version = api_version + self.artman_config = artman_config + self.google_cloud_artifact = f'google-cloud-{service}' + self.grpc_artifact = f'grpc-google-cloud-{service}-{api_version}' + self.proto_artifact = f'proto-google-cloud-{service}-{api_version}' + self.root_directory = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..')) + self.jinja_env = Environment( + loader=FileSystemLoader(self.path('utilities/templates')) + ) + + def path(self, suffix) -> str: + return os.path.join(self.root_directory, suffix) + +def add_to_versions(ctx: Context) -> None: + versions = [] + + # read from versions.txt + versions_path = ctx.path('versions.txt') + with open(versions_path) as f: + for line in f: + version_line = line.strip() + if not version_line or version_line.startswith("#"): + continue + + versions.append(releasetool.ArtifactVersions(version_line)) + + ctx.google_cloud_version = next((v for v in versions if v.module == ctx.google_cloud_artifact), None) + if not ctx.google_cloud_version: + # insert new versions + # use scheduler API for new api versions + version = copy.deepcopy(next(v for v in versions if v.module == 'google-cloud-scheduler')) + version.module = ctx.google_cloud_artifact + versions.append(version) + ctx.google_cloud_version = version + + ctx.proto_version = next((v for v in versions if v.module == ctx.proto_artifact), None) + if not ctx.proto_version: + version = copy.deepcopy(next(v for v in versions if v.module == 'proto-google-cloud-scheduler-v1beta1')) + version.module = ctx.proto_artifact + versions.append(version) + ctx.proto_version = version + + ctx.grpc_version = next((v for v in versions if v.module == ctx.grpc_artifact), None) + if not ctx.grpc_version: + version = copy.deepcopy(next(v for v in versions if v.module == 'grpc-google-cloud-scheduler-v1beta1')) + version.module = ctx.grpc_artifact + versions.append(version) + ctx.grpc_version = version + + # sort by name + versions.sort(key=lambda v: v.module) + + # update versions.txt + with open(versions_path, "w") as f: + f.write("# Format:\n") + f.write("# module:released-version:current-version\n\n") + for version in versions: + f.write("{}\n".format(version)) + + ctx.versions = versions + +def add_module_to_pom(pom: str, module_name: str) -> None: + tree = etree.parse(pom) + root = tree.getroot() + modules = root.find('{http://maven.apache.org/POM/4.0.0}modules') + + new_module = etree.Element('{http://maven.apache.org/POM/4.0.0}module') + new_module.text = module_name + + for i, module in enumerate(modules): + if module.text == module_name: + # print("already added to pom.xml, skipping") + return + + if module.text > module_name: + modules.insert(i, new_module) + break + + tree.write(pom, pretty_print=True, xml_declaration=True, encoding='utf-8') + +def add_dependency_management_to_pom(pom: str, group: str, artifact: str, version: str) -> None: + tree = etree.parse(pom) + root = tree.getroot() + dependencies = root.find('{http://maven.apache.org/POM/4.0.0}dependencyManagement/{http://maven.apache.org/POM/4.0.0}dependencies') + + for dependency in dependencies: + existing = dependency.find('{http://maven.apache.org/POM/4.0.0}artifactId') + if existing is not None: + if existing.text == artifact: + print("already added dependency to pom.xml, skipping") + return + + new_dependency = etree.Element('{http://maven.apache.org/POM/4.0.0}dependency') + group_element = etree.Element('{http://maven.apache.org/POM/4.0.0}groupId') + group_element.text = group + new_dependency.append(group_element) + artifact_element = etree.Element('{http://maven.apache.org/POM/4.0.0}artifactId') + artifact_element.text = artifact + new_dependency.append(artifact_element) + version_element = etree.Element('{http://maven.apache.org/POM/4.0.0}version') + version_element.text = version + new_dependency.append(version_element) + comment = etree.Comment('{x-version-update:' + artifact + ':current}') + new_dependency.append(comment) + dependencies.append(new_dependency) + + tree.write(pom, pretty_print=True, xml_declaration=True, encoding='utf-8') + + +def write_synthfile(ctx: Context) -> None: + template = ctx.jinja_env.get_template('synth.py') + synth = template.stream( + version=ctx.api_version, + service=ctx.service, + config_path=ctx.artman_config, + ) + path = ctx.path(f'google-cloud-clients/{ctx.google_cloud_artifact}/synth.py') + directory = os.path.dirname(path) + if not os.path.isdir(directory): + os.makedirs(directory) + synth.dump(path) + +def write_pom(template: str, path: str, ctx: Context, version: str) -> None: + template = ctx.jinja_env.get_template(template) + pom = template.stream( + api_version=ctx.api_version, + description=ctx.description, + name=ctx.name, + service=ctx.service, + version=version + ) + directory = os.path.dirname(path) + if not os.path.isdir(directory): + os.makedirs(directory) + pom.dump(path) + +def run_synthtool(ctx: Context) -> None: + subprocess.run( + [sys.executable, 'synth.py'], + check=True, + cwd=ctx.path(f'google-cloud-clients/{ctx.google_cloud_artifact}') + ) + +def update_stub_packages(ctx: Context) -> None: + # open google-cloud-clients/pom.xml and fix the Stub packages list + pom = ctx.path('google-cloud-clients/pom.xml') + tree = etree.parse(pom) + + grpc_artifacts = [v.module for v in ctx.versions if v.module.startswith('grpc-')] + stub_classes = [] + for artifact in grpc_artifacts: + m = re.match('grpc-google-cloud-(.*)-(v.*)', artifact) + stub_classes.append(f'com.google.cloud.{m[1]}.{m[2]}.stub') + + for group in tree.findall('.//{http://maven.apache.org/POM/4.0.0}group'): + if group.find('{http://maven.apache.org/POM/4.0.0}title').text == 'Stub packages': + group.find('{http://maven.apache.org/POM/4.0.0}packages').text = ':'.join(stub_classes) + + tree.write(pom, pretty_print=True, xml_declaration=True, encoding='utf-8') + +def write_readme(ctx: Context) -> None: + template = ctx.jinja_env.get_template('README.md') + pom = template.stream( + api_version=ctx.api_version, + description=ctx.description, + name=ctx.name, + service=ctx.service, + version=ctx.google_cloud_version + ) + path = ctx.path(f'google-cloud-clients/{ctx.google_cloud_artifact}/README.md') + directory = os.path.dirname(path) + if not os.path.isdir(directory): + os.makedirs(directory) + pom.dump(path) + + +parser = argparse.ArgumentParser(description='Create a new client') +parser.add_argument('-v', required=True, help='API version (i.e. v1)') +parser.add_argument('-c', required=True, help='Path to config in googleapis/googleapis') +parser.add_argument('-s', required=True, help='Service name') +args = parser.parse_args() + +ctx = Context( + api_version=args.v, + service=args.s, + artman_config=args.c +) + +add_to_versions(ctx) +write_synthfile(ctx) +run_synthtool(ctx) +write_pom( + ctx=ctx, + template="cloud_pom.xml", + path=ctx.path(f'google-cloud-clients/{ctx.google_cloud_artifact}/pom.xml'), + version=ctx.google_cloud_version.current +) +add_module_to_pom( + pom=ctx.path('google-cloud-clients/pom.xml'), + module_name='google-cloud-iamcredentials' +) +add_dependency_management_to_pom( + pom=ctx.path('google-api-grpc/pom.xml'), + group='com.google.api.grpc', + artifact=ctx.proto_artifact, + version=str(ctx.proto_version.current) +) +add_dependency_management_to_pom( + pom=ctx.path('google-api-grpc/pom.xml'), + group='com.google.api.grpc', + artifact=ctx.grpc_artifact, + version=str(ctx.grpc_version.current) +) +add_dependency_management_to_pom( + pom=ctx.path('google-cloud-bom/pom.xml'), + group='com.google.api.grpc', + artifact=ctx.proto_artifact, + version=str(ctx.proto_version.current) +) +add_dependency_management_to_pom( + pom=ctx.path('google-cloud-bom/pom.xml'), + group='com.google.api.grpc', + artifact=ctx.grpc_artifact, + version=str(ctx.grpc_version.current) +) +add_dependency_management_to_pom( + pom=ctx.path('google-cloud-bom/pom.xml'), + group='com.google.cloud', + artifact=ctx.google_cloud_artifact, + version=str(ctx.google_cloud_version.current) +) +write_pom( + ctx=ctx, + template="proto_pom.xml", + path=ctx.path(f'google-api-grpc/{ctx.proto_artifact}/pom.xml'), + version=ctx.proto_version.current +) +write_pom( + ctx=ctx, + template="grpc_pom.xml", + path=ctx.path(f'google-api-grpc/{ctx.grpc_artifact}/pom.xml'), + version=ctx.grpc_version.current +) +add_module_to_pom( + pom=ctx.path('google-api-grpc/pom.xml'), + module_name=ctx.grpc_artifact +) +add_module_to_pom( + pom=ctx.path('google-api-grpc/pom.xml'), + module_name=ctx.proto_artifact +) +update_stub_packages(ctx) +write_readme(ctx) diff --git a/utilities/templates/README.md b/utilities/templates/README.md new file mode 100644 index 000000000000..be22edfd07be --- /dev/null +++ b/utilities/templates/README.md @@ -0,0 +1,100 @@ +Google Cloud Java Client for {{name}} +=================================================== + +Java idiomatic client for [{{name}}][product-overview]. + +[![Kokoro CI](http://storage.googleapis.com/cloud-devrel-public/java/badges/google-cloud-java/master.svg)](http://storage.googleapis.com/cloud-devrel-public/java/badges/google-cloud-java/master.html) +[![Maven](https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-{{service}}.svg)](https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-{{service}}.svg) +[![Codacy Badge](https://api.codacy.com/project/badge/grade/9da006ad7c3a4fe1abd142e77c003917)](https://www.codacy.com/app/mziccard/google-cloud-java) + +- [Product Documentation][product-docs] +- [Client Library Documentation][lib-docs] + +> Note: This client is a work-in-progress, and may occasionally +> make backwards-incompatible changes. + +Quickstart +---------- + +[//]: # ({x-version-update-start:google-cloud-{{service}}:released}) +If you are using Maven, add this to your pom.xml file +```xml + + com.google.cloud + google-cloud-{{service}} + 0.71.0-beta + +``` +If you are using Gradle, add this to your dependencies +```Groovy +compile 'com.google.cloud:google-cloud-{{service}}:0.71.0-beta' +``` +If you are using SBT, add this to your dependencies +```Scala +libraryDependencies += "com.google.cloud" % "google-cloud-{{service}}" % "0.71.0-beta" +``` +[//]: # ({x-version-update-end}) + +Authentication +-------------- + +See the [Authentication](https://github.com/GoogleCloudPlatform/google-cloud-java#authentication) section in the base directory's README. + +About {{name}} +---------------------------- + +[{{name}}][product-overview] FIXME + +See the [{{name}} client library docs][lib-docs] to learn how to use this {{name}} Client Library. + +Getting Started +--------------- +#### Prerequisites +You will need a [Google Developers Console](https://console.developers.google.com/) project with the {{name}} API enabled. [Follow these instructions](https://cloud.google.com/resource-manager/docs/creating-managing-projects) to get your project set up. You will also need to set up the local development environment by [installing the Google Cloud SDK](https://cloud.google.com/sdk/) and running the following commands in command line: `gcloud auth login` and `gcloud config set project [YOUR PROJECT ID]`. + +#### Installation and setup +You'll need to obtain the `google-cloud-{{service}}` library. See the [Quickstart](#quickstart) section to add `google-cloud-{{service}}` as a dependency in your code. + +Troubleshooting +--------------- + +To get help, follow the instructions in the [shared Troubleshooting document](https://github.com/googleapis/google-cloud-common/blob/master/troubleshooting/readme.md#troubleshooting). + +Transport +--------- +{{name}} uses gRPC for the transport layer. + +Java Versions +------------- + +Java 7 or above is required for using this client. + +Versioning +---------- + +This library follows [Semantic Versioning](http://semver.org/). + +It is currently in major version zero (``0.y.z``), which means that anything may change at any time and the public API should not be considered stable. + +Contributing +------------ + +Contributions to this library are always welcome and highly encouraged. + +See `google-cloud`'s [CONTRIBUTING] documentation and the [shared documentation](https://github.com/googleapis/google-cloud-common/blob/master/contributing/readme.md#how-to-contribute-to-gcloud) for more information on how to get started. + +Please note that this project is released with a Contributor Code of Conduct. By participating in this project you agree to abide by its terms. See [Code of Conduct][code-of-conduct] for more information. + +License +------- + +Apache 2.0 - See [LICENSE] for more information. + + +[CONTRIBUTING]:https://github.com/GoogleCloudPlatform/google-cloud-java/blob/master/CONTRIBUTING.md +[code-of-conduct]:https://github.com/GoogleCloudPlatform/google-cloud-java/blob/master/CODE_OF_CONDUCT.md#contributor-code-of-conduct +[LICENSE]: https://github.com/GoogleCloudPlatform/google-cloud-java/blob/master/LICENSE +[cloud-platform]: https://cloud.google.com/ +[product-overview]: FIXME +[product-docs]: FIXME +[lib-docs]: https://googleapis.github.io/google-cloud-java/google-cloud-clients/apidocs/index.html?com/google/cloud/{{service}}/{{api_version}}/package-summary.html diff --git a/utilities/templates/cloud_pom.xml b/utilities/templates/cloud_pom.xml new file mode 100644 index 000000000000..f13b78fffb64 --- /dev/null +++ b/utilities/templates/cloud_pom.xml @@ -0,0 +1,87 @@ + + + 4.0.0 + google-cloud-{{service}} + {{version}} + jar + {{name}} + https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-{{service}} + + {{description}} + + + com.google.cloud + google-cloud-clients + {{version}} + + + google-cloud-{{service}} + + + + ${project.groupId} + google-cloud-core + + + ${project.groupId} + google-cloud-core-grpc + + + com.google.api.grpc + proto-google-cloud-{{service}}-{{api_version}} + + + com.google.api.grpc + grpc-google-cloud-{{service}}-{{api_version}} + + + io.grpc + grpc-netty-shaded + + + io.grpc + grpc-stub + + + io.grpc + grpc-auth + + + ${project.groupId} + google-cloud-core + test-jar + test + + + junit + junit + test + + + org.easymock + easymock + test + + + org.objenesis + objenesis + test + + + com.google.truth + truth + test + + + com.google.api.grpc + grpc-google-iam-v1 + test + + + com.google.api + gax-grpc + testlib + test + + + diff --git a/utilities/templates/grpc_pom.xml b/utilities/templates/grpc_pom.xml new file mode 100644 index 000000000000..5fd6917880b5 --- /dev/null +++ b/utilities/templates/grpc_pom.xml @@ -0,0 +1,31 @@ + + 4.0.0 + grpc-google-cloud-{{service}}-{{api_version}} + {{version}} + grpc-google-cloud-{{service}}-{{api_version}} + GRPC library for grpc-google-cloud-{{service}}-{{api_version}} + + com.google.api.grpc + google-api-grpc + {{version}} + + + + io.grpc + grpc-stub + compile + + + io.grpc + grpc-protobuf + compile + + + com.google.api.grpc + proto-google-cloud-{{service}}-{{api_version}} + compile + + + diff --git a/utilities/templates/proto_pom.xml b/utilities/templates/proto_pom.xml new file mode 100644 index 000000000000..7567238d0488 --- /dev/null +++ b/utilities/templates/proto_pom.xml @@ -0,0 +1,36 @@ + + 4.0.0 + proto-google-cloud-{{service}}-{{api_version}} + {{version}} + proto-google-cloud-{{service}}-{{api_version}} + PROTO library for proto-google-cloud-{{service}}-{{api_version}} + + com.google.api.grpc + google-api-grpc + {{version}} + + + + com.google.protobuf + protobuf-java + compile + + + com.google.api + api-common + compile + + + com.google.api.grpc + proto-google-common-protos + compile + + + com.google.api.grpc + proto-google-iam-v1 + compile + + + diff --git a/utilities/templates/synth.py b/utilities/templates/synth.py new file mode 100644 index 000000000000..c0a3ba98b1cb --- /dev/null +++ b/utilities/templates/synth.py @@ -0,0 +1,35 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This script is used to synthesize generated parts of this library.""" + +import synthtool as s +import synthtool.gcp as gcp + +gapic = gcp.GAPICGenerator() +common_templates = gcp.CommonTemplates() + +versions = ['{{version}}'] +service = '{{service}}' + +for version in versions: + library = gapic.java_library( + service=service, + version=version, + config_path=f'{{config_path}}', + artman_output_name='') + + s.copy(library / f'gapic-google-cloud-{service}-{version}/src', 'src') + s.copy(library / f'grpc-google-cloud-{service}-{version}/src', f'../../google-api-grpc/grpc-google-cloud-{service}-{version}/src') + s.copy(library / f'proto-google-cloud-{service}-{version}/src', f'../../google-api-grpc/proto-google-cloud-{service}-{version}/src') From 0eb718a9d746e6e08f10df8a7afb5630af2ad208 Mon Sep 17 00:00:00 2001 From: Jeff Ching Date: Wed, 5 Dec 2018 15:59:49 -0800 Subject: [PATCH 2/9] Add requirements.txt, convert to use attrs --- utilities/new_client.py | 18 ++++++++++-------- utilities/requirements.txt | 5 +++++ 2 files changed, 15 insertions(+), 8 deletions(-) create mode 100644 utilities/requirements.txt diff --git a/utilities/new_client.py b/utilities/new_client.py index ee758e199ad2..23fcce24a9b2 100644 --- a/utilities/new_client.py +++ b/utilities/new_client.py @@ -1,4 +1,5 @@ -# Copyright 2018 Google LLC +#!/usr/bin/env python3 +# Copyright 2018 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,11 +21,14 @@ import re import sys import subprocess + +import attr from jinja2 import Environment, FileSystemLoader from lxml import etree from releasetool.commands.start import java as releasetool from typing import List +@attr.s(auto_attribs=True) class Context: service: str = None api_version: str = None @@ -39,14 +43,12 @@ class Context: description: str = "FIXME" name: str = "FIXME" versions: List[str] = None + jinja_env: Environment = None - def __init__(self, service: str, api_version: str, artman_config: str): - self.service = service - self.api_version = api_version - self.artman_config = artman_config - self.google_cloud_artifact = f'google-cloud-{service}' - self.grpc_artifact = f'grpc-google-cloud-{service}-{api_version}' - self.proto_artifact = f'proto-google-cloud-{service}-{api_version}' + def __attrs_post_init__(self): + self.google_cloud_artifact = f'google-cloud-{self.service}' + self.grpc_artifact = f'grpc-google-cloud-{self.service}-{self.api_version}' + self.proto_artifact = f'proto-google-cloud-{self.service}-{self.api_version}' self.root_directory = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..')) self.jinja_env = Environment( loader=FileSystemLoader(self.path('utilities/templates')) diff --git a/utilities/requirements.txt b/utilities/requirements.txt new file mode 100644 index 000000000000..6ce1600b1b6e --- /dev/null +++ b/utilities/requirements.txt @@ -0,0 +1,5 @@ +attr +gcp-releasetool +jinja2 +lxml~=4.2 +typing~=3.6 From 00116c0f20e23d91cac03b7949a2d0a86df4bc51 Mon Sep 17 00:00:00 2001 From: Jeff Ching Date: Wed, 5 Dec 2018 16:09:12 -0800 Subject: [PATCH 3/9] Start new artifact versions at 0.0.0-alpha. Next release will bump version --- utilities/new_client.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/utilities/new_client.py b/utilities/new_client.py index 23fcce24a9b2..9592056148c9 100644 --- a/utilities/new_client.py +++ b/utilities/new_client.py @@ -55,9 +55,11 @@ def __attrs_post_init__(self): ) def path(self, suffix) -> str: + """Helper to generate a path from the repository root.""" return os.path.join(self.root_directory, suffix) def add_to_versions(ctx: Context) -> None: + """Add the new artifact's versions to the versions.txt manifest.""" versions = [] # read from versions.txt @@ -70,28 +72,21 @@ def add_to_versions(ctx: Context) -> None: versions.append(releasetool.ArtifactVersions(version_line)) + # Add new versions unless the artifacts already exist in the versions.txt manifest ctx.google_cloud_version = next((v for v in versions if v.module == ctx.google_cloud_artifact), None) if not ctx.google_cloud_version: - # insert new versions - # use scheduler API for new api versions - version = copy.deepcopy(next(v for v in versions if v.module == 'google-cloud-scheduler')) - version.module = ctx.google_cloud_artifact - versions.append(version) - ctx.google_cloud_version = version + ctx.google_cloud_version = releasetool.ArtifactVersions(f"{ctx.google_cloud_artifact}:0.0.0-alpha:0.0.1-alpha-SNAPSHOT") + versions.append(ctx.google_cloud_version) ctx.proto_version = next((v for v in versions if v.module == ctx.proto_artifact), None) if not ctx.proto_version: - version = copy.deepcopy(next(v for v in versions if v.module == 'proto-google-cloud-scheduler-v1beta1')) - version.module = ctx.proto_artifact - versions.append(version) - ctx.proto_version = version + ctx.proto_version = releasetool.ArtifactVersions(f"{ctx.proto_artifact}:0.0.0-alpha:0.0.1-alpha-SNAPSHOT") + versions.append(ctx.proto_version) ctx.grpc_version = next((v for v in versions if v.module == ctx.grpc_artifact), None) if not ctx.grpc_version: - version = copy.deepcopy(next(v for v in versions if v.module == 'grpc-google-cloud-scheduler-v1beta1')) - version.module = ctx.grpc_artifact - versions.append(version) - ctx.grpc_version = version + ctx.grpc_version = releasetool.ArtifactVersions(f"{ctx.grpc_artifact}:0.0.0-alpha:0.0.1-alpha-SNAPSHOT") + versions.append(ctx.grpc_version) # sort by name versions.sort(key=lambda v: v.module) From 2a68d9d84584803cde294dcf483035bfdc102b9d Mon Sep 17 00:00:00 2001 From: Jeff Ching Date: Wed, 5 Dec 2018 16:18:12 -0800 Subject: [PATCH 4/9] Switch to pathlib --- utilities/new_client.py | 48 +++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/utilities/new_client.py b/utilities/new_client.py index 9592056148c9..f4b8628a47db 100644 --- a/utilities/new_client.py +++ b/utilities/new_client.py @@ -16,8 +16,8 @@ # Usage: python new_client.py -v -s -c import argparse -import copy import os +import pathlib import re import sys import subprocess @@ -39,7 +39,8 @@ class Context: grpc_version: releasetool.Version = None proto_artifact: str = None proto_version: releasetool.Version = None - root_directory: str = None + root_directory: pathlib.Path = \ + pathlib.Path(os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'))) description: str = "FIXME" name: str = "FIXME" versions: List[str] = None @@ -49,21 +50,16 @@ def __attrs_post_init__(self): self.google_cloud_artifact = f'google-cloud-{self.service}' self.grpc_artifact = f'grpc-google-cloud-{self.service}-{self.api_version}' self.proto_artifact = f'proto-google-cloud-{self.service}-{self.api_version}' - self.root_directory = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..')) self.jinja_env = Environment( - loader=FileSystemLoader(self.path('utilities/templates')) + loader=FileSystemLoader(str(self.root_directory / 'utilities/templates')) ) - def path(self, suffix) -> str: - """Helper to generate a path from the repository root.""" - return os.path.join(self.root_directory, suffix) - def add_to_versions(ctx: Context) -> None: """Add the new artifact's versions to the versions.txt manifest.""" versions = [] # read from versions.txt - versions_path = ctx.path('versions.txt') + versions_path = ctx.root_directory / 'versions.txt' with open(versions_path) as f: for line in f: version_line = line.strip() @@ -155,11 +151,11 @@ def write_synthfile(ctx: Context) -> None: service=ctx.service, config_path=ctx.artman_config, ) - path = ctx.path(f'google-cloud-clients/{ctx.google_cloud_artifact}/synth.py') + path = ctx.root_directory / 'google-cloud-clients' / ctx.google_cloud_artifact / 'synth.py' directory = os.path.dirname(path) if not os.path.isdir(directory): os.makedirs(directory) - synth.dump(path) + synth.dump(str(path)) def write_pom(template: str, path: str, ctx: Context, version: str) -> None: template = ctx.jinja_env.get_template(template) @@ -173,18 +169,18 @@ def write_pom(template: str, path: str, ctx: Context, version: str) -> None: directory = os.path.dirname(path) if not os.path.isdir(directory): os.makedirs(directory) - pom.dump(path) + pom.dump(str(path)) def run_synthtool(ctx: Context) -> None: subprocess.run( [sys.executable, 'synth.py'], check=True, - cwd=ctx.path(f'google-cloud-clients/{ctx.google_cloud_artifact}') + cwd=ctx.root_directory / 'google-cloud-clients' / ctx.google_cloud_artifact ) def update_stub_packages(ctx: Context) -> None: # open google-cloud-clients/pom.xml and fix the Stub packages list - pom = ctx.path('google-cloud-clients/pom.xml') + pom = ctx.root_directory / 'google-cloud-clients/pom.xml' tree = etree.parse(pom) grpc_artifacts = [v.module for v in ctx.versions if v.module.startswith('grpc-')] @@ -208,7 +204,7 @@ def write_readme(ctx: Context) -> None: service=ctx.service, version=ctx.google_cloud_version ) - path = ctx.path(f'google-cloud-clients/{ctx.google_cloud_artifact}/README.md') + path = ctx.root_directory / 'google-cloud-clients' / ctx.google_cloud_artifact / 'README.md' directory = os.path.dirname(path) if not os.path.isdir(directory): os.makedirs(directory) @@ -233,39 +229,39 @@ def write_readme(ctx: Context) -> None: write_pom( ctx=ctx, template="cloud_pom.xml", - path=ctx.path(f'google-cloud-clients/{ctx.google_cloud_artifact}/pom.xml'), + path=ctx.root_directory / 'google-cloud-clients' / ctx.google_cloud_artifact / 'pom.xml', version=ctx.google_cloud_version.current ) add_module_to_pom( - pom=ctx.path('google-cloud-clients/pom.xml'), + pom=ctx.root_directory / 'google-cloud-clients/pom.xml', module_name='google-cloud-iamcredentials' ) add_dependency_management_to_pom( - pom=ctx.path('google-api-grpc/pom.xml'), + pom=ctx.root_directory / 'google-api-grpc/pom.xml', group='com.google.api.grpc', artifact=ctx.proto_artifact, version=str(ctx.proto_version.current) ) add_dependency_management_to_pom( - pom=ctx.path('google-api-grpc/pom.xml'), + pom=ctx.root_directory / 'google-api-grpc/pom.xml', group='com.google.api.grpc', artifact=ctx.grpc_artifact, version=str(ctx.grpc_version.current) ) add_dependency_management_to_pom( - pom=ctx.path('google-cloud-bom/pom.xml'), + pom=ctx.root_directory / 'google-cloud-bom/pom.xml', group='com.google.api.grpc', artifact=ctx.proto_artifact, version=str(ctx.proto_version.current) ) add_dependency_management_to_pom( - pom=ctx.path('google-cloud-bom/pom.xml'), + pom=ctx.root_directory / 'google-cloud-bom/pom.xml', group='com.google.api.grpc', artifact=ctx.grpc_artifact, version=str(ctx.grpc_version.current) ) add_dependency_management_to_pom( - pom=ctx.path('google-cloud-bom/pom.xml'), + pom=ctx.root_directory / 'google-cloud-bom/pom.xml', group='com.google.cloud', artifact=ctx.google_cloud_artifact, version=str(ctx.google_cloud_version.current) @@ -273,21 +269,21 @@ def write_readme(ctx: Context) -> None: write_pom( ctx=ctx, template="proto_pom.xml", - path=ctx.path(f'google-api-grpc/{ctx.proto_artifact}/pom.xml'), + path=ctx.root_directory / 'google-api-grpc' / ctx.proto_artifact / 'pom.xml', version=ctx.proto_version.current ) write_pom( ctx=ctx, template="grpc_pom.xml", - path=ctx.path(f'google-api-grpc/{ctx.grpc_artifact}/pom.xml'), + path=ctx.root_directory / 'google-api-grpc' / ctx.grpc_artifact / 'pom.xml', version=ctx.grpc_version.current ) add_module_to_pom( - pom=ctx.path('google-api-grpc/pom.xml'), + pom=ctx.root_directory / 'google-api-grpc/pom.xml', module_name=ctx.grpc_artifact ) add_module_to_pom( - pom=ctx.path('google-api-grpc/pom.xml'), + pom=ctx.root_directory / 'google-api-grpc/pom.xml', module_name=ctx.proto_artifact ) update_stub_packages(ctx) From 86149010465330d6a2812dc90215ff886cf45358 Mon Sep 17 00:00:00 2001 From: Jeff Ching Date: Wed, 5 Dec 2018 16:19:36 -0800 Subject: [PATCH 5/9] Move main execution into main() --- utilities/new_client.py | 157 ++++++++++++++++++++-------------------- 1 file changed, 80 insertions(+), 77 deletions(-) diff --git a/utilities/new_client.py b/utilities/new_client.py index f4b8628a47db..929fbd829e31 100644 --- a/utilities/new_client.py +++ b/utilities/new_client.py @@ -210,81 +210,84 @@ def write_readme(ctx: Context) -> None: os.makedirs(directory) pom.dump(path) +def main(): + parser = argparse.ArgumentParser(description='Create a new client') + parser.add_argument('-v', required=True, help='API version (i.e. v1)') + parser.add_argument('-c', required=True, help='Path to config in googleapis/googleapis') + parser.add_argument('-s', required=True, help='Service name') + args = parser.parse_args() + + ctx = Context( + api_version=args.v, + service=args.s, + artman_config=args.c + ) + + add_to_versions(ctx) + write_synthfile(ctx) + run_synthtool(ctx) + write_pom( + ctx=ctx, + template="cloud_pom.xml", + path=ctx.root_directory / 'google-cloud-clients' / ctx.google_cloud_artifact / 'pom.xml', + version=ctx.google_cloud_version.current + ) + add_module_to_pom( + pom=ctx.root_directory / 'google-cloud-clients/pom.xml', + module_name='google-cloud-iamcredentials' + ) + add_dependency_management_to_pom( + pom=ctx.root_directory / 'google-api-grpc/pom.xml', + group='com.google.api.grpc', + artifact=ctx.proto_artifact, + version=str(ctx.proto_version.current) + ) + add_dependency_management_to_pom( + pom=ctx.root_directory / 'google-api-grpc/pom.xml', + group='com.google.api.grpc', + artifact=ctx.grpc_artifact, + version=str(ctx.grpc_version.current) + ) + add_dependency_management_to_pom( + pom=ctx.root_directory / 'google-cloud-bom/pom.xml', + group='com.google.api.grpc', + artifact=ctx.proto_artifact, + version=str(ctx.proto_version.current) + ) + add_dependency_management_to_pom( + pom=ctx.root_directory / 'google-cloud-bom/pom.xml', + group='com.google.api.grpc', + artifact=ctx.grpc_artifact, + version=str(ctx.grpc_version.current) + ) + add_dependency_management_to_pom( + pom=ctx.root_directory / 'google-cloud-bom/pom.xml', + group='com.google.cloud', + artifact=ctx.google_cloud_artifact, + version=str(ctx.google_cloud_version.current) + ) + write_pom( + ctx=ctx, + template="proto_pom.xml", + path=ctx.root_directory / 'google-api-grpc' / ctx.proto_artifact / 'pom.xml', + version=ctx.proto_version.current + ) + write_pom( + ctx=ctx, + template="grpc_pom.xml", + path=ctx.root_directory / 'google-api-grpc' / ctx.grpc_artifact / 'pom.xml', + version=ctx.grpc_version.current + ) + add_module_to_pom( + pom=ctx.root_directory / 'google-api-grpc/pom.xml', + module_name=ctx.grpc_artifact + ) + add_module_to_pom( + pom=ctx.root_directory / 'google-api-grpc/pom.xml', + module_name=ctx.proto_artifact + ) + update_stub_packages(ctx) + write_readme(ctx) -parser = argparse.ArgumentParser(description='Create a new client') -parser.add_argument('-v', required=True, help='API version (i.e. v1)') -parser.add_argument('-c', required=True, help='Path to config in googleapis/googleapis') -parser.add_argument('-s', required=True, help='Service name') -args = parser.parse_args() - -ctx = Context( - api_version=args.v, - service=args.s, - artman_config=args.c -) - -add_to_versions(ctx) -write_synthfile(ctx) -run_synthtool(ctx) -write_pom( - ctx=ctx, - template="cloud_pom.xml", - path=ctx.root_directory / 'google-cloud-clients' / ctx.google_cloud_artifact / 'pom.xml', - version=ctx.google_cloud_version.current -) -add_module_to_pom( - pom=ctx.root_directory / 'google-cloud-clients/pom.xml', - module_name='google-cloud-iamcredentials' -) -add_dependency_management_to_pom( - pom=ctx.root_directory / 'google-api-grpc/pom.xml', - group='com.google.api.grpc', - artifact=ctx.proto_artifact, - version=str(ctx.proto_version.current) -) -add_dependency_management_to_pom( - pom=ctx.root_directory / 'google-api-grpc/pom.xml', - group='com.google.api.grpc', - artifact=ctx.grpc_artifact, - version=str(ctx.grpc_version.current) -) -add_dependency_management_to_pom( - pom=ctx.root_directory / 'google-cloud-bom/pom.xml', - group='com.google.api.grpc', - artifact=ctx.proto_artifact, - version=str(ctx.proto_version.current) -) -add_dependency_management_to_pom( - pom=ctx.root_directory / 'google-cloud-bom/pom.xml', - group='com.google.api.grpc', - artifact=ctx.grpc_artifact, - version=str(ctx.grpc_version.current) -) -add_dependency_management_to_pom( - pom=ctx.root_directory / 'google-cloud-bom/pom.xml', - group='com.google.cloud', - artifact=ctx.google_cloud_artifact, - version=str(ctx.google_cloud_version.current) -) -write_pom( - ctx=ctx, - template="proto_pom.xml", - path=ctx.root_directory / 'google-api-grpc' / ctx.proto_artifact / 'pom.xml', - version=ctx.proto_version.current -) -write_pom( - ctx=ctx, - template="grpc_pom.xml", - path=ctx.root_directory / 'google-api-grpc' / ctx.grpc_artifact / 'pom.xml', - version=ctx.grpc_version.current -) -add_module_to_pom( - pom=ctx.root_directory / 'google-api-grpc/pom.xml', - module_name=ctx.grpc_artifact -) -add_module_to_pom( - pom=ctx.root_directory / 'google-api-grpc/pom.xml', - module_name=ctx.proto_artifact -) -update_stub_packages(ctx) -write_readme(ctx) +if __name__ == "__main__": + main() From 4da03b50bb60925e66665f83cd9fa9bbf1471a60 Mon Sep 17 00:00:00 2001 From: Jeff Ching Date: Wed, 5 Dec 2018 16:24:39 -0800 Subject: [PATCH 6/9] Standardize quotes to double-quotes --- utilities/new_client.py | 106 ++++++++++++++++++++-------------------- 1 file changed, 52 insertions(+), 54 deletions(-) diff --git a/utilities/new_client.py b/utilities/new_client.py index 929fbd829e31..2c45ea0d180b 100644 --- a/utilities/new_client.py +++ b/utilities/new_client.py @@ -13,8 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Usage: python new_client.py -v -s -c - import argparse import os import pathlib @@ -40,18 +38,18 @@ class Context: proto_artifact: str = None proto_version: releasetool.Version = None root_directory: pathlib.Path = \ - pathlib.Path(os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'))) + pathlib.Path(os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), ".."))) description: str = "FIXME" name: str = "FIXME" versions: List[str] = None jinja_env: Environment = None def __attrs_post_init__(self): - self.google_cloud_artifact = f'google-cloud-{self.service}' - self.grpc_artifact = f'grpc-google-cloud-{self.service}-{self.api_version}' - self.proto_artifact = f'proto-google-cloud-{self.service}-{self.api_version}' + self.google_cloud_artifact = f"google-cloud-{self.service}" + self.grpc_artifact = f"grpc-google-cloud-{self.service}-{self.api_version}" + self.proto_artifact = f"proto-google-cloud-{self.service}-{self.api_version}" self.jinja_env = Environment( - loader=FileSystemLoader(str(self.root_directory / 'utilities/templates')) + loader=FileSystemLoader(str(self.root_directory / "utilities/templates")) ) def add_to_versions(ctx: Context) -> None: @@ -59,7 +57,7 @@ def add_to_versions(ctx: Context) -> None: versions = [] # read from versions.txt - versions_path = ctx.root_directory / 'versions.txt' + versions_path = ctx.root_directory / "versions.txt" with open(versions_path) as f: for line in f: version_line = line.strip() @@ -99,9 +97,9 @@ def add_to_versions(ctx: Context) -> None: def add_module_to_pom(pom: str, module_name: str) -> None: tree = etree.parse(pom) root = tree.getroot() - modules = root.find('{http://maven.apache.org/POM/4.0.0}modules') + modules = root.find("{http://maven.apache.org/POM/4.0.0}modules") - new_module = etree.Element('{http://maven.apache.org/POM/4.0.0}module') + new_module = etree.Element("{http://maven.apache.org/POM/4.0.0}module") new_module.text = module_name for i, module in enumerate(modules): @@ -113,45 +111,45 @@ def add_module_to_pom(pom: str, module_name: str) -> None: modules.insert(i, new_module) break - tree.write(pom, pretty_print=True, xml_declaration=True, encoding='utf-8') + tree.write(pom, pretty_print=True, xml_declaration=True, encoding="utf-8") def add_dependency_management_to_pom(pom: str, group: str, artifact: str, version: str) -> None: tree = etree.parse(pom) root = tree.getroot() - dependencies = root.find('{http://maven.apache.org/POM/4.0.0}dependencyManagement/{http://maven.apache.org/POM/4.0.0}dependencies') + dependencies = root.find("{http://maven.apache.org/POM/4.0.0}dependencyManagement/{http://maven.apache.org/POM/4.0.0}dependencies") for dependency in dependencies: - existing = dependency.find('{http://maven.apache.org/POM/4.0.0}artifactId') + existing = dependency.find("{http://maven.apache.org/POM/4.0.0}artifactId") if existing is not None: if existing.text == artifact: print("already added dependency to pom.xml, skipping") return - new_dependency = etree.Element('{http://maven.apache.org/POM/4.0.0}dependency') - group_element = etree.Element('{http://maven.apache.org/POM/4.0.0}groupId') + new_dependency = etree.Element("{http://maven.apache.org/POM/4.0.0}dependency") + group_element = etree.Element("{http://maven.apache.org/POM/4.0.0}groupId") group_element.text = group new_dependency.append(group_element) - artifact_element = etree.Element('{http://maven.apache.org/POM/4.0.0}artifactId') + artifact_element = etree.Element("{http://maven.apache.org/POM/4.0.0}artifactId") artifact_element.text = artifact new_dependency.append(artifact_element) - version_element = etree.Element('{http://maven.apache.org/POM/4.0.0}version') + version_element = etree.Element("{http://maven.apache.org/POM/4.0.0}version") version_element.text = version new_dependency.append(version_element) - comment = etree.Comment('{x-version-update:' + artifact + ':current}') + comment = etree.Comment("{x-version-update:" + artifact + ":current}") new_dependency.append(comment) dependencies.append(new_dependency) - tree.write(pom, pretty_print=True, xml_declaration=True, encoding='utf-8') + tree.write(pom, pretty_print=True, xml_declaration=True, encoding="utf-8") def write_synthfile(ctx: Context) -> None: - template = ctx.jinja_env.get_template('synth.py') + template = ctx.jinja_env.get_template("synth.py") synth = template.stream( version=ctx.api_version, service=ctx.service, config_path=ctx.artman_config, ) - path = ctx.root_directory / 'google-cloud-clients' / ctx.google_cloud_artifact / 'synth.py' + path = ctx.root_directory / "google-cloud-clients" / ctx.google_cloud_artifact / "synth.py" directory = os.path.dirname(path) if not os.path.isdir(directory): os.makedirs(directory) @@ -173,30 +171,30 @@ def write_pom(template: str, path: str, ctx: Context, version: str) -> None: def run_synthtool(ctx: Context) -> None: subprocess.run( - [sys.executable, 'synth.py'], + [sys.executable, "synth.py"], check=True, - cwd=ctx.root_directory / 'google-cloud-clients' / ctx.google_cloud_artifact + cwd=ctx.root_directory / "google-cloud-clients" / ctx.google_cloud_artifact ) def update_stub_packages(ctx: Context) -> None: # open google-cloud-clients/pom.xml and fix the Stub packages list - pom = ctx.root_directory / 'google-cloud-clients/pom.xml' + pom = ctx.root_directory / "google-cloud-clients/pom.xml" tree = etree.parse(pom) - grpc_artifacts = [v.module for v in ctx.versions if v.module.startswith('grpc-')] + grpc_artifacts = [v.module for v in ctx.versions if v.module.startswith("grpc-")] stub_classes = [] for artifact in grpc_artifacts: - m = re.match('grpc-google-cloud-(.*)-(v.*)', artifact) - stub_classes.append(f'com.google.cloud.{m[1]}.{m[2]}.stub') + m = re.match("grpc-google-cloud-(.*)-(v.*)", artifact) + stub_classes.append(f"com.google.cloud.{m[1]}.{m[2]}.stub") - for group in tree.findall('.//{http://maven.apache.org/POM/4.0.0}group'): - if group.find('{http://maven.apache.org/POM/4.0.0}title').text == 'Stub packages': - group.find('{http://maven.apache.org/POM/4.0.0}packages').text = ':'.join(stub_classes) + for group in tree.findall(".//{http://maven.apache.org/POM/4.0.0}group"): + if group.find("{http://maven.apache.org/POM/4.0.0}title").text == "Stub packages": + group.find("{http://maven.apache.org/POM/4.0.0}packages").text = ":".join(stub_classes) - tree.write(pom, pretty_print=True, xml_declaration=True, encoding='utf-8') + tree.write(pom, pretty_print=True, xml_declaration=True, encoding="utf-8") def write_readme(ctx: Context) -> None: - template = ctx.jinja_env.get_template('README.md') + template = ctx.jinja_env.get_template("README.md") pom = template.stream( api_version=ctx.api_version, description=ctx.description, @@ -204,17 +202,17 @@ def write_readme(ctx: Context) -> None: service=ctx.service, version=ctx.google_cloud_version ) - path = ctx.root_directory / 'google-cloud-clients' / ctx.google_cloud_artifact / 'README.md' + path = ctx.root_directory / "google-cloud-clients" / ctx.google_cloud_artifact / "README.md" directory = os.path.dirname(path) if not os.path.isdir(directory): os.makedirs(directory) pom.dump(path) def main(): - parser = argparse.ArgumentParser(description='Create a new client') - parser.add_argument('-v', required=True, help='API version (i.e. v1)') - parser.add_argument('-c', required=True, help='Path to config in googleapis/googleapis') - parser.add_argument('-s', required=True, help='Service name') + parser = argparse.ArgumentParser(description="Create a new client") + parser.add_argument("-v", required=True, help="API version (i.e. v1)") + parser.add_argument("-c", required=True, help="Path to config in googleapis/googleapis") + parser.add_argument("-s", required=True, help="Service name") args = parser.parse_args() ctx = Context( @@ -229,61 +227,61 @@ def main(): write_pom( ctx=ctx, template="cloud_pom.xml", - path=ctx.root_directory / 'google-cloud-clients' / ctx.google_cloud_artifact / 'pom.xml', + path=ctx.root_directory / "google-cloud-clients" / ctx.google_cloud_artifact / "pom.xml", version=ctx.google_cloud_version.current ) add_module_to_pom( - pom=ctx.root_directory / 'google-cloud-clients/pom.xml', - module_name='google-cloud-iamcredentials' + pom=ctx.root_directory / "google-cloud-clients/pom.xml", + module_name="google-cloud-iamcredentials" ) add_dependency_management_to_pom( - pom=ctx.root_directory / 'google-api-grpc/pom.xml', - group='com.google.api.grpc', + pom=ctx.root_directory / "google-api-grpc/pom.xml", + group="com.google.api.grpc", artifact=ctx.proto_artifact, version=str(ctx.proto_version.current) ) add_dependency_management_to_pom( - pom=ctx.root_directory / 'google-api-grpc/pom.xml', - group='com.google.api.grpc', + pom=ctx.root_directory / "google-api-grpc/pom.xml", + group="com.google.api.grpc", artifact=ctx.grpc_artifact, version=str(ctx.grpc_version.current) ) add_dependency_management_to_pom( - pom=ctx.root_directory / 'google-cloud-bom/pom.xml', - group='com.google.api.grpc', + pom=ctx.root_directory / "google-cloud-bom/pom.xml", + group="com.google.api.grpc", artifact=ctx.proto_artifact, version=str(ctx.proto_version.current) ) add_dependency_management_to_pom( - pom=ctx.root_directory / 'google-cloud-bom/pom.xml', - group='com.google.api.grpc', + pom=ctx.root_directory / "google-cloud-bom/pom.xml", + group="com.google.api.grpc", artifact=ctx.grpc_artifact, version=str(ctx.grpc_version.current) ) add_dependency_management_to_pom( - pom=ctx.root_directory / 'google-cloud-bom/pom.xml', - group='com.google.cloud', + pom=ctx.root_directory / "google-cloud-bom/pom.xml", + group="com.google.cloud", artifact=ctx.google_cloud_artifact, version=str(ctx.google_cloud_version.current) ) write_pom( ctx=ctx, template="proto_pom.xml", - path=ctx.root_directory / 'google-api-grpc' / ctx.proto_artifact / 'pom.xml', + path=ctx.root_directory / "google-api-grpc" / ctx.proto_artifact / "pom.xml", version=ctx.proto_version.current ) write_pom( ctx=ctx, template="grpc_pom.xml", - path=ctx.root_directory / 'google-api-grpc' / ctx.grpc_artifact / 'pom.xml', + path=ctx.root_directory / "google-api-grpc" / ctx.grpc_artifact / "pom.xml", version=ctx.grpc_version.current ) add_module_to_pom( - pom=ctx.root_directory / 'google-api-grpc/pom.xml', + pom=ctx.root_directory / "google-api-grpc/pom.xml", module_name=ctx.grpc_artifact ) add_module_to_pom( - pom=ctx.root_directory / 'google-api-grpc/pom.xml', + pom=ctx.root_directory / "google-api-grpc/pom.xml", module_name=ctx.proto_artifact ) update_stub_packages(ctx) From f5a869cc3c8985c74c69bf67b60ee1c82b9844d9 Mon Sep 17 00:00:00 2001 From: Jeff Ching Date: Wed, 5 Dec 2018 16:33:04 -0800 Subject: [PATCH 7/9] Update docstrings --- utilities/new_client.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/utilities/new_client.py b/utilities/new_client.py index 2c45ea0d180b..b64938a950ad 100644 --- a/utilities/new_client.py +++ b/utilities/new_client.py @@ -95,6 +95,12 @@ def add_to_versions(ctx: Context) -> None: ctx.versions = versions def add_module_to_pom(pom: str, module_name: str) -> None: + """ + Insert a new name into a pom.xml file. + + This assumes that the modules are in alphabetical order and inserts it + in the correct place + """ tree = etree.parse(pom) root = tree.getroot() modules = root.find("{http://maven.apache.org/POM/4.0.0}modules") @@ -114,6 +120,7 @@ def add_module_to_pom(pom: str, module_name: str) -> None: tree.write(pom, pretty_print=True, xml_declaration=True, encoding="utf-8") def add_dependency_management_to_pom(pom: str, group: str, artifact: str, version: str) -> None: + """Adds a ... section into a pom.xml file.""" tree = etree.parse(pom) root = tree.getroot() dependencies = root.find("{http://maven.apache.org/POM/4.0.0}dependencyManagement/{http://maven.apache.org/POM/4.0.0}dependencies") @@ -143,6 +150,7 @@ def add_dependency_management_to_pom(pom: str, group: str, artifact: str, versio def write_synthfile(ctx: Context) -> None: + """Creates a synth.py file from a template.""" template = ctx.jinja_env.get_template("synth.py") synth = template.stream( version=ctx.api_version, @@ -156,6 +164,7 @@ def write_synthfile(ctx: Context) -> None: synth.dump(str(path)) def write_pom(template: str, path: str, ctx: Context, version: str) -> None: + """Creates a pom.xml file from a template.""" template = ctx.jinja_env.get_template(template) pom = template.stream( api_version=ctx.api_version, @@ -170,6 +179,7 @@ def write_pom(template: str, path: str, ctx: Context, version: str) -> None: pom.dump(str(path)) def run_synthtool(ctx: Context) -> None: + """Runs synthtool for the initial client generation.""" subprocess.run( [sys.executable, "synth.py"], check=True, @@ -177,6 +187,7 @@ def run_synthtool(ctx: Context) -> None: ) def update_stub_packages(ctx: Context) -> None: + """Updates the pom.xml documentation section to scrub certain packages.""" # open google-cloud-clients/pom.xml and fix the Stub packages list pom = ctx.root_directory / "google-cloud-clients/pom.xml" tree = etree.parse(pom) @@ -194,6 +205,7 @@ def update_stub_packages(ctx: Context) -> None: tree.write(pom, pretty_print=True, xml_declaration=True, encoding="utf-8") def write_readme(ctx: Context) -> None: + """Creates a README.md from a template.""" template = ctx.jinja_env.get_template("README.md") pom = template.stream( api_version=ctx.api_version, From 499ec21f73f4268fb4bcc0c1c12e78b8311a29d2 Mon Sep 17 00:00:00 2001 From: Jeff Ching Date: Wed, 12 Dec 2018 13:50:06 -0800 Subject: [PATCH 8/9] Fix Path vs. string types when reading/writing files --- utilities/new_client.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/utilities/new_client.py b/utilities/new_client.py index b64938a950ad..9b3ddcb1ae9a 100644 --- a/utilities/new_client.py +++ b/utilities/new_client.py @@ -94,14 +94,14 @@ def add_to_versions(ctx: Context) -> None: ctx.versions = versions -def add_module_to_pom(pom: str, module_name: str) -> None: +def add_module_to_pom(pom: pathlib.Path, module_name: str) -> None: """ Insert a new name into a pom.xml file. This assumes that the modules are in alphabetical order and inserts it in the correct place """ - tree = etree.parse(pom) + tree = etree.parse(str(pom)) root = tree.getroot() modules = root.find("{http://maven.apache.org/POM/4.0.0}modules") @@ -117,11 +117,11 @@ def add_module_to_pom(pom: str, module_name: str) -> None: modules.insert(i, new_module) break - tree.write(pom, pretty_print=True, xml_declaration=True, encoding="utf-8") + tree.write(str(pom), pretty_print=True, xml_declaration=True, encoding="utf-8") -def add_dependency_management_to_pom(pom: str, group: str, artifact: str, version: str) -> None: +def add_dependency_management_to_pom(pom: pathlib.Path, group: str, artifact: str, version: str) -> None: """Adds a ... section into a pom.xml file.""" - tree = etree.parse(pom) + tree = etree.parse(str(pom)) root = tree.getroot() dependencies = root.find("{http://maven.apache.org/POM/4.0.0}dependencyManagement/{http://maven.apache.org/POM/4.0.0}dependencies") @@ -146,7 +146,7 @@ def add_dependency_management_to_pom(pom: str, group: str, artifact: str, versio new_dependency.append(comment) dependencies.append(new_dependency) - tree.write(pom, pretty_print=True, xml_declaration=True, encoding="utf-8") + tree.write(str(pom), pretty_print=True, xml_declaration=True, encoding="utf-8") def write_synthfile(ctx: Context) -> None: @@ -190,7 +190,7 @@ def update_stub_packages(ctx: Context) -> None: """Updates the pom.xml documentation section to scrub certain packages.""" # open google-cloud-clients/pom.xml and fix the Stub packages list pom = ctx.root_directory / "google-cloud-clients/pom.xml" - tree = etree.parse(pom) + tree = etree.parse(str(pom)) grpc_artifacts = [v.module for v in ctx.versions if v.module.startswith("grpc-")] stub_classes = [] @@ -202,7 +202,7 @@ def update_stub_packages(ctx: Context) -> None: if group.find("{http://maven.apache.org/POM/4.0.0}title").text == "Stub packages": group.find("{http://maven.apache.org/POM/4.0.0}packages").text = ":".join(stub_classes) - tree.write(pom, pretty_print=True, xml_declaration=True, encoding="utf-8") + tree.write(str(pom), pretty_print=True, xml_declaration=True, encoding="utf-8") def write_readme(ctx: Context) -> None: """Creates a README.md from a template.""" @@ -218,7 +218,7 @@ def write_readme(ctx: Context) -> None: directory = os.path.dirname(path) if not os.path.isdir(directory): os.makedirs(directory) - pom.dump(path) + pom.dump(str(path)) def main(): parser = argparse.ArgumentParser(description="Create a new client") From 5333a892588639086fd8edd7008fa90ac1828320 Mon Sep 17 00:00:00 2001 From: Jeff Ching Date: Wed, 12 Dec 2018 15:27:43 -0800 Subject: [PATCH 9/9] Update synth.py template to include the java formatter --- utilities/templates/synth.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/utilities/templates/synth.py b/utilities/templates/synth.py index c0a3ba98b1cb..1c97d7f6fb34 100644 --- a/utilities/templates/synth.py +++ b/utilities/templates/synth.py @@ -16,6 +16,7 @@ import synthtool as s import synthtool.gcp as gcp +import synthtool.languages.java as java gapic = gcp.GAPICGenerator() common_templates = gcp.CommonTemplates() @@ -33,3 +34,7 @@ s.copy(library / f'gapic-google-cloud-{service}-{version}/src', 'src') s.copy(library / f'grpc-google-cloud-{service}-{version}/src', f'../../google-api-grpc/grpc-google-cloud-{service}-{version}/src') s.copy(library / f'proto-google-cloud-{service}-{version}/src', f'../../google-api-grpc/proto-google-cloud-{service}-{version}/src') + + java.format_code('./src') + java.format_code(f'../../google-api-grpc/grpc-google-cloud-{service}-{version}/src') + java.format_code(f'../../google-api-grpc/proto-google-cloud-{service}-{version}/src')