From 32dc915e6315c11630cc9429a61846c012ec7c0b Mon Sep 17 00:00:00 2001 From: Marcin Podolski Date: Fri, 18 Jan 2019 18:11:53 +0100 Subject: [PATCH] Python code generation for protobufs and gRPC (#6974) ### Problem I would like to have a task, to generate python code from protobufs and grpc services. ### Solution There is a new codegen task **grpcio-run**, to execute python's grpcio library and generate python code from .proto files. ### Example: Respectful examples can be found in [/examples/src/python/example/grpcio](/examples/src/python/example/grpcio) to create a gRPC server execute ```./pants run examples/src/python/example/grpcio/server``` and when it's running, run client example: ```./pants run examples/src/python/example/grpcio/client``` generated code can be found as usual in pants output directory: ```/.pants.d/gen/grpcio-run/current/examples.src.protobuf.org.pantsbuild.example.service.service/current/org/pantsbuild/example/service``` --- 3rdparty/python/requirements.txt | 2 + .../pantsbuild/example/grpcio/imports/BUILD | 11 ++++ .../example/grpcio/imports/imports.proto | 16 +++++ .../pantsbuild/example/grpcio/service/BUILD | 10 +++ .../example/grpcio/service/service.proto | 12 ++++ .../src/python/example/grpcio/__init__.py | 0 .../src/python/example/grpcio/client/BUILD | 22 +++++++ .../python/example/grpcio/client/__init__.py | 0 .../example/grpcio/client/imports_client.py | 33 ++++++++++ .../example/grpcio/client/service_client.py | 31 +++++++++ .../src/python/example/grpcio/server/BUILD | 14 ++++ .../python/example/grpcio/server/__init__.py | 0 .../python/example/grpcio/server/server.py | 52 +++++++++++++++ src/python/pants/backend/codegen/BUILD | 1 + .../pants/backend/codegen/grpcio/__init__.py | 0 .../pants/backend/codegen/grpcio/python/BUILD | 19 ++++++ .../backend/codegen/grpcio/python/__init__.py | 0 .../backend/codegen/grpcio/python/grpcio.py | 18 +++++ .../codegen/grpcio/python/grpcio_prep.py | 25 +++++++ .../codegen/grpcio/python/grpcio_run.py | 61 +++++++++++++++++ .../grpcio/python/python_grpcio_library.py | 14 ++++ .../backend/codegen/grpcio/python/register.py | 24 +++++++ src/python/pants/backend/codegen/register.py | 6 ++ src/python/pants/option/global_options.py | 1 + .../pants_test/backend/codegen/grpcio/BUILD | 12 ++++ .../backend/codegen/grpcio/__init__.py | 0 .../codegen/grpcio/grpcio_test_base.py | 35 ++++++++++ .../grpcio/test_multiple_grpcio_gen.py | 66 +++++++++++++++++++ .../codegen/grpcio/test_single_grpcio_gen.py | 42 ++++++++++++ 29 files changed, 527 insertions(+) create mode 100644 examples/src/protobuf/org/pantsbuild/example/grpcio/imports/BUILD create mode 100644 examples/src/protobuf/org/pantsbuild/example/grpcio/imports/imports.proto create mode 100644 examples/src/protobuf/org/pantsbuild/example/grpcio/service/BUILD create mode 100644 examples/src/protobuf/org/pantsbuild/example/grpcio/service/service.proto create mode 100644 examples/src/python/example/grpcio/__init__.py create mode 100644 examples/src/python/example/grpcio/client/BUILD create mode 100644 examples/src/python/example/grpcio/client/__init__.py create mode 100644 examples/src/python/example/grpcio/client/imports_client.py create mode 100644 examples/src/python/example/grpcio/client/service_client.py create mode 100644 examples/src/python/example/grpcio/server/BUILD create mode 100644 examples/src/python/example/grpcio/server/__init__.py create mode 100644 examples/src/python/example/grpcio/server/server.py create mode 100644 src/python/pants/backend/codegen/grpcio/__init__.py create mode 100644 src/python/pants/backend/codegen/grpcio/python/BUILD create mode 100644 src/python/pants/backend/codegen/grpcio/python/__init__.py create mode 100644 src/python/pants/backend/codegen/grpcio/python/grpcio.py create mode 100644 src/python/pants/backend/codegen/grpcio/python/grpcio_prep.py create mode 100644 src/python/pants/backend/codegen/grpcio/python/grpcio_run.py create mode 100644 src/python/pants/backend/codegen/grpcio/python/python_grpcio_library.py create mode 100644 src/python/pants/backend/codegen/grpcio/python/register.py create mode 100644 tests/python/pants_test/backend/codegen/grpcio/BUILD create mode 100644 tests/python/pants_test/backend/codegen/grpcio/__init__.py create mode 100644 tests/python/pants_test/backend/codegen/grpcio/grpcio_test_base.py create mode 100644 tests/python/pants_test/backend/codegen/grpcio/test_multiple_grpcio_gen.py create mode 100644 tests/python/pants_test/backend/codegen/grpcio/test_single_grpcio_gen.py diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt index 676ecdffe16..ad02f1c74d7 100644 --- a/3rdparty/python/requirements.txt +++ b/3rdparty/python/requirements.txt @@ -10,6 +10,7 @@ fasteners==0.14.1 faulthandler==2.6 ; python_version<'3' future==0.16.0 futures==3.0.5 ; python_version<'3' +grpcio==1.16.1 Markdown==2.1.1 mock==2.0.0 packaging==16.8 @@ -17,6 +18,7 @@ parameterized==0.6.1 pathspec==0.5.9 pex==1.5.3 psutil==4.3.0 +protobuf==3.6.1 pycodestyle==2.4.0 pyflakes==2.0.0 Pygments==2.3.1 diff --git a/examples/src/protobuf/org/pantsbuild/example/grpcio/imports/BUILD b/examples/src/protobuf/org/pantsbuild/example/grpcio/imports/BUILD new file mode 100644 index 00000000000..a1f03d55678 --- /dev/null +++ b/examples/src/protobuf/org/pantsbuild/example/grpcio/imports/BUILD @@ -0,0 +1,11 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_grpcio_library( + sources=['imports.proto'], + dependencies=[ + '3rdparty/python:protobuf', + 'examples/src/protobuf/org/pantsbuild/example/grpcio/service' + ] +) diff --git a/examples/src/protobuf/org/pantsbuild/example/grpcio/imports/imports.proto b/examples/src/protobuf/org/pantsbuild/example/grpcio/imports/imports.proto new file mode 100644 index 00000000000..db686d7eb58 --- /dev/null +++ b/examples/src/protobuf/org/pantsbuild/example/grpcio/imports/imports.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; +package org.pantsbuild.example.grpcio.imports; + +import "org/pantsbuild/example/grpcio/service/service.proto"; + +service ImportsService { + rpc HelloImports(HelloImportsRequest) returns (HelloImportsReply) {} +} + +message HelloImportsRequest { + org.pantsbuild.example.grpcio.service.HelloRequest hello_request = 1; +} + +message HelloImportsReply{ + org.pantsbuild.example.grpcio.service.HelloReply hello_reply = 1; +} \ No newline at end of file diff --git a/examples/src/protobuf/org/pantsbuild/example/grpcio/service/BUILD b/examples/src/protobuf/org/pantsbuild/example/grpcio/service/BUILD new file mode 100644 index 00000000000..0a2e213156e --- /dev/null +++ b/examples/src/protobuf/org/pantsbuild/example/grpcio/service/BUILD @@ -0,0 +1,10 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_grpcio_library( + sources=['service.proto'], + dependencies=[ + '3rdparty/python:protobuf', + ] +) diff --git a/examples/src/protobuf/org/pantsbuild/example/grpcio/service/service.proto b/examples/src/protobuf/org/pantsbuild/example/grpcio/service/service.proto new file mode 100644 index 00000000000..3ecac6e787d --- /dev/null +++ b/examples/src/protobuf/org/pantsbuild/example/grpcio/service/service.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; +package org.pantsbuild.example.grpcio.service; + +service ExampleService { + rpc Hello(HelloRequest) returns (HelloReply) {} +} +message HelloRequest { + string action = 1; +} +message HelloReply { + string response = 1; +} \ No newline at end of file diff --git a/examples/src/python/example/grpcio/__init__.py b/examples/src/python/example/grpcio/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/examples/src/python/example/grpcio/client/BUILD b/examples/src/python/example/grpcio/client/BUILD new file mode 100644 index 00000000000..f9d2e3dccf3 --- /dev/null +++ b/examples/src/python/example/grpcio/client/BUILD @@ -0,0 +1,22 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_binary( + name='service', + dependencies=[ + '3rdparty/python:grpcio', + + 'examples/src/protobuf/org/pantsbuild/example/grpcio/service', + ], + source='service_client.py', +) + +python_binary( + name='imports', + dependencies=[ + '3rdparty/python:grpcio', + + 'examples/src/protobuf/org/pantsbuild/example/grpcio/imports', + ], + source='imports_client.py', +) \ No newline at end of file diff --git a/examples/src/python/example/grpcio/client/__init__.py b/examples/src/python/example/grpcio/client/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/examples/src/python/example/grpcio/client/imports_client.py b/examples/src/python/example/grpcio/client/imports_client.py new file mode 100644 index 00000000000..9ba3a69b561 --- /dev/null +++ b/examples/src/python/example/grpcio/client/imports_client.py @@ -0,0 +1,33 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import grpc +from org.pantsbuild.example.grpcio.imports import imports_pb2, imports_pb2_grpc +from org.pantsbuild.example.grpcio.service import service_pb2 + + +def run_example(): + print('hello world from grpcio imports_client!') + with grpc.insecure_channel('localhost:50051') as channel: + stub = imports_pb2_grpc.ImportsServiceStub(channel) + try: + + hello_request = service_pb2.HelloRequest(action='hello with imports') + request = imports_pb2.HelloImportsRequest(hello_request=hello_request) + reply = stub.HelloImports(request) + except grpc.RpcError as error: + if error.code() == grpc.StatusCode.UNAVAILABLE: + print("[ERROR] Connection to server is unavailable. You should create a server instance first.") + print("To start a gRPC server, execute: `./pants run examples/src/python/example/grpcio/server`") + else: + print('An error occured! Error code: [{}] Error details: [{}]'.format(error.code(), error.details())) + else: + print(reply.hello_reply.response) + print('[SUCCESS]') + + +if __name__ == '__main__': + run_example() diff --git a/examples/src/python/example/grpcio/client/service_client.py b/examples/src/python/example/grpcio/client/service_client.py new file mode 100644 index 00000000000..80648d80c88 --- /dev/null +++ b/examples/src/python/example/grpcio/client/service_client.py @@ -0,0 +1,31 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import grpc +from org.pantsbuild.example.grpcio.service import service_pb2, service_pb2_grpc + + +def run_example(): + print('hello world from grpcio service_client!') + with grpc.insecure_channel('localhost:50051') as channel: + stub = service_pb2_grpc.ExampleServiceStub(channel) + try: + hello_response = stub.Hello(service_pb2.HelloRequest(action='hello')) + bye_response = stub.Hello(service_pb2.HelloRequest(action='bye')) + except grpc.RpcError as error: + if error.code() == grpc.StatusCode.UNAVAILABLE: + print("[ERROR] Connection to server is unavailable. You should create a server instance first.") + print("To start a gRPC server, execute: `./pants run examples/src/python/example/grpcio/server`") + else: + print('An error occured! Error code: [{}] Error details: [{}]'.format(error.code(), error.details())) + else: + print(hello_response) + print(bye_response) + print('[SUCCESS]') + + +if __name__ == '__main__': + run_example() diff --git a/examples/src/python/example/grpcio/server/BUILD b/examples/src/python/example/grpcio/server/BUILD new file mode 100644 index 00000000000..502260c376b --- /dev/null +++ b/examples/src/python/example/grpcio/server/BUILD @@ -0,0 +1,14 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_binary( + name='server', + dependencies=[ + '3rdparty/python:grpcio', + + 'examples/src/protobuf/org/pantsbuild/example/grpcio/service', + 'examples/src/protobuf/org/pantsbuild/example/grpcio/imports', + ], + source='server.py', +) \ No newline at end of file diff --git a/examples/src/python/example/grpcio/server/__init__.py b/examples/src/python/example/grpcio/server/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/examples/src/python/example/grpcio/server/server.py b/examples/src/python/example/grpcio/server/server.py new file mode 100644 index 00000000000..d437cb69308 --- /dev/null +++ b/examples/src/python/example/grpcio/server/server.py @@ -0,0 +1,52 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import time +from concurrent import futures + +import grpc +from org.pantsbuild.example.grpcio.imports import imports_pb2, imports_pb2_grpc +from org.pantsbuild.example.grpcio.service import service_pb2, service_pb2_grpc + + +class ExampleHelloServer(service_pb2_grpc.ExampleServiceServicer): + + def Hello(self, request, context): + print('request with action: [{}]'.format(request.action)) + reply = service_pb2.HelloReply() + reply.response = '{} from server!'.format(request.action) + return reply + + +class ImportsServiceServer(imports_pb2_grpc.ImportsServiceServicer): + + def HelloImports(self, request, context): + print('request with action: [{}]'.format(request.hello_request.action)) + hello_reply = service_pb2.HelloReply(response='{} from imports server!'.format(request.hello_request.action)) + reply = imports_pb2.HelloImportsReply(hello_reply=hello_reply) + return reply + + +def run_server(): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=5)) + + service_pb2_grpc.add_ExampleServiceServicer_to_server(ExampleHelloServer(), server) + imports_pb2_grpc.add_ImportsServiceServicer_to_server(ImportsServiceServer(), server) + + server.add_insecure_port('[::]:50051') + server.start() + + print('Server is running...') + print('(hit Ctrl+C to stop)') + try: + while True: + time.sleep(10) + except KeyboardInterrupt: + server.stop(0) + + +if __name__ == '__main__': + run_server() diff --git a/src/python/pants/backend/codegen/BUILD b/src/python/pants/backend/codegen/BUILD index db44ab59403..37ba08c556e 100644 --- a/src/python/pants/backend/codegen/BUILD +++ b/src/python/pants/backend/codegen/BUILD @@ -13,6 +13,7 @@ python_library( 'src/python/pants/backend/codegen/thrift/java', 'src/python/pants/backend/codegen/thrift/python', 'src/python/pants/backend/codegen/wire/java', + 'src/python/pants/backend/codegen/grpcio/python', 'src/python/pants/build_graph', 'src/python/pants/goal:task_registrar', ] diff --git a/src/python/pants/backend/codegen/grpcio/__init__.py b/src/python/pants/backend/codegen/grpcio/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/codegen/grpcio/python/BUILD b/src/python/pants/backend/codegen/grpcio/python/BUILD new file mode 100644 index 00000000000..c431d8ba732 --- /dev/null +++ b/src/python/pants/backend/codegen/grpcio/python/BUILD @@ -0,0 +1,19 @@ +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_library( + dependencies = [ + '3rdparty/python:future', + '3rdparty/python:pex', + 'src/python/pants/backend/python:plugin', + 'src/python/pants/backend/python/subsystems', + 'src/python/pants/backend/python/targets', + 'src/python/pants/base:workunit', + 'src/python/pants/base:build_environment', + 'src/python/pants/base:exceptions', + 'src/python/pants/build_graph', + 'src/python/pants/subsystem', + 'src/python/pants/util:contextutil', + 'src/python/pants/task', + ], +) \ No newline at end of file diff --git a/src/python/pants/backend/codegen/grpcio/python/__init__.py b/src/python/pants/backend/codegen/grpcio/python/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/python/pants/backend/codegen/grpcio/python/grpcio.py b/src/python/pants/backend/codegen/grpcio/python/grpcio.py new file mode 100644 index 00000000000..3ef4a800403 --- /dev/null +++ b/src/python/pants/backend/codegen/grpcio/python/grpcio.py @@ -0,0 +1,18 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +from pants.backend.python.subsystems.python_tool_base import PythonToolBase + + +class Grpcio(PythonToolBase): + grpcio_version = '1.17.1' + + options_scope = 'grpcio' + default_requirements = [ + 'grpcio-tools=={}'.format(grpcio_version), + 'grpcio=={}'.format(grpcio_version), + ] + default_entry_point = 'grpc_tools.protoc' diff --git a/src/python/pants/backend/codegen/grpcio/python/grpcio_prep.py b/src/python/pants/backend/codegen/grpcio/python/grpcio_prep.py new file mode 100644 index 00000000000..6a576ae4d27 --- /dev/null +++ b/src/python/pants/backend/codegen/grpcio/python/grpcio_prep.py @@ -0,0 +1,25 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +from pants.backend.codegen.grpcio.python.grpcio import Grpcio +from pants.backend.codegen.grpcio.python.python_grpcio_library import PythonGrpcioLibrary +from pants.backend.python.tasks.python_tool_prep_base import PythonToolInstance, PythonToolPrepBase + + +class GrpcioInstance(PythonToolInstance): + pass + + +class GrpcioPrep(PythonToolPrepBase): + tool_subsystem_cls = Grpcio + tool_instance_cls = GrpcioInstance + + def execute(self): + targets = self.get_targets(lambda target: isinstance(target, PythonGrpcioLibrary)) + if not targets: + return 0 + + super(GrpcioPrep, self).execute() diff --git a/src/python/pants/backend/codegen/grpcio/python/grpcio_run.py b/src/python/pants/backend/codegen/grpcio/python/grpcio_run.py new file mode 100644 index 00000000000..9d3f9001ef2 --- /dev/null +++ b/src/python/pants/backend/codegen/grpcio/python/grpcio_run.py @@ -0,0 +1,61 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import functools +import logging + +from pants.backend.codegen.grpcio.python.grpcio_prep import GrpcioPrep +from pants.backend.codegen.grpcio.python.python_grpcio_library import PythonGrpcioLibrary +from pants.backend.python.targets.python_library import PythonLibrary +from pants.base.build_environment import get_buildroot +from pants.base.exceptions import TaskError +from pants.base.workunit import WorkUnitLabel +from pants.task.simple_codegen_task import SimpleCodegenTask +from pants.util.contextutil import pushd +from pants.util.memo import memoized_property + + +class GrpcioRun(SimpleCodegenTask): + """Task to compile protobuf into python code""" + + gentarget_type = PythonGrpcioLibrary + sources_globs = ('**/*',) + + @classmethod + def prepare(cls, options, round_manager): + super(GrpcioRun, cls).prepare(options, round_manager) + round_manager.require_data(GrpcioPrep.tool_instance_cls) + + def synthetic_target_type(self, target): + return PythonLibrary + + @memoized_property + def _grpcio_binary(self): + return self.context.products.get_data(GrpcioPrep.tool_instance_cls) + + def execute_codegen(self, target, target_workdir): + args = self.build_args(target, target_workdir) + logging.debug("Executing grpcio code generation with args: [{}]".format(args)) + + with pushd(get_buildroot()): + workunit_factory = functools.partial(self.context.new_workunit, + name='run-grpcio', + labels=[WorkUnitLabel.TOOL, WorkUnitLabel.LINT]) + cmdline, exit_code = self._grpcio_binary.run(workunit_factory, args) + if exit_code != 0: + raise TaskError('{} ... exited non-zero ({}).'.format(cmdline, exit_code), + exit_code=exit_code) + logging.info("Grpcio finished code generation into: [{}]".format(target_workdir)) + + def build_args(self, target, target_workdir): + proto_path = '--proto_path={0}'.format(target.target_base) + python_out = '--python_out={0}'.format(target_workdir) + grpc_python_out = '--grpc_python_out={0}'.format(target_workdir) + + args = [python_out, grpc_python_out, proto_path] + + args.extend(target.sources_relative_to_buildroot()) + return args diff --git a/src/python/pants/backend/codegen/grpcio/python/python_grpcio_library.py b/src/python/pants/backend/codegen/grpcio/python/python_grpcio_library.py new file mode 100644 index 00000000000..505f60834f5 --- /dev/null +++ b/src/python/pants/backend/codegen/grpcio/python/python_grpcio_library.py @@ -0,0 +1,14 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +from pants.backend.python.targets.python_target import PythonTarget + + +class PythonGrpcioLibrary(PythonTarget): + """A Python library generated from Protocol Buffer IDL files.""" + + def __init__(self, sources=None, **kwargs): + super(PythonGrpcioLibrary, self).__init__(sources=sources, **kwargs) diff --git a/src/python/pants/backend/codegen/grpcio/python/register.py b/src/python/pants/backend/codegen/grpcio/python/register.py new file mode 100644 index 00000000000..e092aac4503 --- /dev/null +++ b/src/python/pants/backend/codegen/grpcio/python/register.py @@ -0,0 +1,24 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +from pants.backend.codegen.grpcio.python.grpcio_prep import GrpcioPrep +from pants.backend.codegen.grpcio.python.grpcio_run import GrpcioRun +from pants.backend.codegen.grpcio.python.python_grpcio_library import PythonGrpcioLibrary +from pants.build_graph.build_file_aliases import BuildFileAliases +from pants.goal.task_registrar import TaskRegistrar as task + + +def build_file_aliases(): + return BuildFileAliases( + targets={ + 'python_grpcio_library': PythonGrpcioLibrary, + } + ) + + +def register_goals(): + task(name='grpcio-prep', action=GrpcioPrep).install('gen') + task(name='grpcio-run', action=GrpcioRun).install('gen') diff --git a/src/python/pants/backend/codegen/register.py b/src/python/pants/backend/codegen/register.py index 5c7093fcc75..247857d43cf 100644 --- a/src/python/pants/backend/codegen/register.py +++ b/src/python/pants/backend/codegen/register.py @@ -8,6 +8,9 @@ from pants.backend.codegen.antlr.java.java_antlr_library import JavaAntlrLibrary from pants.backend.codegen.antlr.python.antlr_py_gen import AntlrPyGen from pants.backend.codegen.antlr.python.python_antlr_library import PythonAntlrLibrary +from pants.backend.codegen.grpcio.python.grpcio_prep import GrpcioPrep +from pants.backend.codegen.grpcio.python.grpcio_run import GrpcioRun +from pants.backend.codegen.grpcio.python.python_grpcio_library import PythonGrpcioLibrary from pants.backend.codegen.jaxb.jaxb_gen import JaxbGen from pants.backend.codegen.jaxb.jaxb_library import JaxbLibrary from pants.backend.codegen.protobuf.java.java_protobuf_library import JavaProtobufLibrary @@ -34,6 +37,7 @@ def build_file_aliases(): 'java_wire_library': JavaWireLibrary, 'python_antlr_library': PythonAntlrLibrary, 'python_thrift_library': PythonThriftLibrary, + 'python_grpcio_library': PythonGrpcioLibrary, 'jaxb_library': JaxbLibrary, } ) @@ -42,6 +46,8 @@ def build_file_aliases(): def register_goals(): task(name='thrift-java', action=ApacheThriftJavaGen).install('gen') task(name='thrift-py', action=ApacheThriftPyGen).install('gen') + task(name='grpcio-prep', action=GrpcioPrep).install('gen') + task(name='grpcio-run', action=GrpcioRun).install('gen') task(name='protoc', action=ProtobufGen).install('gen') task(name='antlr-java', action=AntlrJavaGen).install('gen') task(name='antlr-py', action=AntlrPyGen).install('gen') diff --git a/src/python/pants/option/global_options.py b/src/python/pants/option/global_options.py index 0ef7f44d9e7..31160095c0e 100644 --- a/src/python/pants/option/global_options.py +++ b/src/python/pants/option/global_options.py @@ -143,6 +143,7 @@ def register_bootstrap_options(cls, register): 'pants.backend.codegen.ragel.java', 'pants.backend.codegen.thrift.java', 'pants.backend.codegen.thrift.python', + 'pants.backend.codegen.grpcio.python', 'pants.backend.codegen.wire.java', 'pants.backend.project_info'], help='Load backends from these packages that are already on the path. ' diff --git a/tests/python/pants_test/backend/codegen/grpcio/BUILD b/tests/python/pants_test/backend/codegen/grpcio/BUILD new file mode 100644 index 00000000000..a1a91d59b4c --- /dev/null +++ b/tests/python/pants_test/backend/codegen/grpcio/BUILD @@ -0,0 +1,12 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +python_tests( + dependencies=[ + 'src/python/pants/backend/python/targets:python', + 'src/python/pants/base:build_environment', + 'tests/python/pants_test:task_test_base', + ], + sources=globs('*.py') +) diff --git a/tests/python/pants_test/backend/codegen/grpcio/__init__.py b/tests/python/pants_test/backend/codegen/grpcio/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/python/pants_test/backend/codegen/grpcio/grpcio_test_base.py b/tests/python/pants_test/backend/codegen/grpcio/grpcio_test_base.py new file mode 100644 index 00000000000..aa05e27032e --- /dev/null +++ b/tests/python/pants_test/backend/codegen/grpcio/grpcio_test_base.py @@ -0,0 +1,35 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +import os + +from pants.backend.codegen.grpcio.python.grpcio_prep import GrpcioPrep +from pants.backend.codegen.grpcio.python.grpcio_run import GrpcioRun +from pants.backend.python.targets.python_library import PythonLibrary +from pants_test.task_test_base import TaskTestBase + + +class GrpcioTestBase(TaskTestBase): + + @classmethod + def task_type(cls): + return GrpcioRun + + def generate_grpcio_targets(self, python_grpcio_library): + grpcio_prep_task_type = self.synthesize_task_subtype(GrpcioPrep, 'gp') + context = self.context(for_task_types=[grpcio_prep_task_type], + target_roots=[python_grpcio_library]) + + grpcio_prep = grpcio_prep_task_type(context, os.path.join(self.pants_workdir, 'gp')) + grpcio_prep.execute() + + grpcio_gen = self.create_task(context) + grpcio_gen.execute() + + def is_synthetic_python_library(target): + return isinstance(target, PythonLibrary) and target.is_synthetic + + return context.targets(predicate=is_synthetic_python_library) diff --git a/tests/python/pants_test/backend/codegen/grpcio/test_multiple_grpcio_gen.py b/tests/python/pants_test/backend/codegen/grpcio/test_multiple_grpcio_gen.py new file mode 100644 index 00000000000..d1ff56ab5e3 --- /dev/null +++ b/tests/python/pants_test/backend/codegen/grpcio/test_multiple_grpcio_gen.py @@ -0,0 +1,66 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +from textwrap import dedent + +from pants.backend.codegen.grpcio.python.python_grpcio_library import PythonGrpcioLibrary +from pants_test.backend.codegen.grpcio.grpcio_test_base import GrpcioTestBase + + +class GrpcioGenTest(GrpcioTestBase): + + def test_multiple_dependent_protobufs(self): + # given + self.create_file('src/grpcio/com/foo/foo_example.proto', contents=dedent(""" + syntax = "proto3"; + package com.foo; + + service FooService { + rpc Foo(FooRequest) returns (FooReply) {} + } + message FooRequest { + string foo = 1; + } + message FooReply { + string bar = 1; + } + """)) + self.create_file('src/grpcio/com/bar/bar_example.proto', contents=dedent(""" + syntax = "proto3"; + package com.bar; + + import "com/foo/foo_example.proto"; + + service BarService { + rpc Bar(BarRequest) returns (BarReply) {} + } + message BarRequest { + com.foo.FooRequest foo_request = 1; + } + message BarReply { + com.foo.FooReply foo_reply = 1; + } + """)) + foo_target = self.make_target(spec='src/grpcio/com/foo:example', + target_type=PythonGrpcioLibrary, + sources=['foo_example.proto']) + bar_target = self.make_target(spec='src/grpcio/com/bar:example', + target_type=PythonGrpcioLibrary, + sources=['bar_example.proto'], + dependencies=[foo_target]) + + # when + synthetic_target = self.generate_grpcio_targets(bar_target) + + # then + self.assertIsNotNone(synthetic_target) + self.assertEqual(2, len(synthetic_target)) + self.assertEqual({'com/foo/foo_example_pb2_grpc.py', + 'com/foo/foo_example_pb2.py'}, + set(synthetic_target[0].sources_relative_to_source_root())) + self.assertEqual({'com/bar/bar_example_pb2_grpc.py', + 'com/bar/bar_example_pb2.py'}, + set(synthetic_target[1].sources_relative_to_source_root())) diff --git a/tests/python/pants_test/backend/codegen/grpcio/test_single_grpcio_gen.py b/tests/python/pants_test/backend/codegen/grpcio/test_single_grpcio_gen.py new file mode 100644 index 00000000000..45cdff12026 --- /dev/null +++ b/tests/python/pants_test/backend/codegen/grpcio/test_single_grpcio_gen.py @@ -0,0 +1,42 @@ +# coding=utf-8 +# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). +# Licensed under the Apache License, Version 2.0 (see LICENSE). + +from __future__ import absolute_import, division, print_function, unicode_literals + +from textwrap import dedent + +from pants.backend.codegen.grpcio.python.python_grpcio_library import PythonGrpcioLibrary +from pants_test.backend.codegen.grpcio.grpcio_test_base import GrpcioTestBase + + +class GrpcioMultipleGenTest(GrpcioTestBase): + + def test_single_protobuf(self): + # given + self.create_file('src/grpcio/com/example/example.proto', contents=dedent(""" + syntax = "proto3"; + package com.example; + + service FooService { + rpc Foo(FooRequest) returns (FooReply) {} + } + message FooRequest { + string foo = 1; + } + message FooReply { + string bar = 1; + } + """)) + example_target = self.make_target(spec='src/grpcio/com/example:example', + target_type=PythonGrpcioLibrary, + sources=['example.proto']) + + # when + synthetic_target = self.generate_grpcio_targets(example_target) + + # then + self.assertEqual(1, len(synthetic_target)) + self.assertEqual({'com/example/example_pb2_grpc.py', + 'com/example/example_pb2.py'}, + set(synthetic_target[0].sources_relative_to_source_root()))