forked from pantsbuild/pants
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Python code generation for protobufs and gRPC (pantsbuild#6974)
### Problem I would like to have a task, to generate python code from protobufs and grpc services. ### Solution There is a new codegen task **grpcio-run**, to execute python's grpcio library <https://grpc.io/> and generate python code from .proto files. ### Example: Respectful examples can be found in [/examples/src/python/example/grpcio](/examples/src/python/example/grpcio) to create a gRPC server execute ```./pants run examples/src/python/example/grpcio/server``` and when it's running, run client example: ```./pants run examples/src/python/example/grpcio/client``` generated code can be found as usual in pants output directory: ```/.pants.d/gen/grpcio-run/current/examples.src.protobuf.org.pantsbuild.example.service.service/current/org/pantsbuild/example/service```
- Loading branch information
Showing
29 changed files
with
527 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
11 changes: 11 additions & 0 deletions
11
examples/src/protobuf/org/pantsbuild/example/grpcio/imports/BUILD
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
# coding=utf-8 | ||
# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). | ||
# Licensed under the Apache License, Version 2.0 (see LICENSE). | ||
|
||
python_grpcio_library( | ||
sources=['imports.proto'], | ||
dependencies=[ | ||
'3rdparty/python:protobuf', | ||
'examples/src/protobuf/org/pantsbuild/example/grpcio/service' | ||
] | ||
) |
16 changes: 16 additions & 0 deletions
16
examples/src/protobuf/org/pantsbuild/example/grpcio/imports/imports.proto
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
syntax = "proto3"; | ||
package org.pantsbuild.example.grpcio.imports; | ||
|
||
import "org/pantsbuild/example/grpcio/service/service.proto"; | ||
|
||
service ImportsService { | ||
rpc HelloImports(HelloImportsRequest) returns (HelloImportsReply) {} | ||
} | ||
|
||
message HelloImportsRequest { | ||
org.pantsbuild.example.grpcio.service.HelloRequest hello_request = 1; | ||
} | ||
|
||
message HelloImportsReply{ | ||
org.pantsbuild.example.grpcio.service.HelloReply hello_reply = 1; | ||
} |
10 changes: 10 additions & 0 deletions
10
examples/src/protobuf/org/pantsbuild/example/grpcio/service/BUILD
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
# coding=utf-8 | ||
# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). | ||
# Licensed under the Apache License, Version 2.0 (see LICENSE). | ||
|
||
python_grpcio_library( | ||
sources=['service.proto'], | ||
dependencies=[ | ||
'3rdparty/python:protobuf', | ||
] | ||
) |
12 changes: 12 additions & 0 deletions
12
examples/src/protobuf/org/pantsbuild/example/grpcio/service/service.proto
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
syntax = "proto3"; | ||
package org.pantsbuild.example.grpcio.service; | ||
|
||
service ExampleService { | ||
rpc Hello(HelloRequest) returns (HelloReply) {} | ||
} | ||
message HelloRequest { | ||
string action = 1; | ||
} | ||
message HelloReply { | ||
string response = 1; | ||
} |
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). | ||
# Licensed under the Apache License, Version 2.0 (see LICENSE). | ||
|
||
python_binary( | ||
name='service', | ||
dependencies=[ | ||
'3rdparty/python:grpcio', | ||
|
||
'examples/src/protobuf/org/pantsbuild/example/grpcio/service', | ||
], | ||
source='service_client.py', | ||
) | ||
|
||
python_binary( | ||
name='imports', | ||
dependencies=[ | ||
'3rdparty/python:grpcio', | ||
|
||
'examples/src/protobuf/org/pantsbuild/example/grpcio/imports', | ||
], | ||
source='imports_client.py', | ||
) |
Empty file.
33 changes: 33 additions & 0 deletions
33
examples/src/python/example/grpcio/client/imports_client.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,33 @@ | ||
# coding=utf-8 | ||
# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). | ||
# Licensed under the Apache License, Version 2.0 (see LICENSE). | ||
|
||
from __future__ import absolute_import, division, print_function, unicode_literals | ||
|
||
import grpc | ||
from org.pantsbuild.example.grpcio.imports import imports_pb2, imports_pb2_grpc | ||
from org.pantsbuild.example.grpcio.service import service_pb2 | ||
|
||
|
||
def run_example(): | ||
print('hello world from grpcio imports_client!') | ||
with grpc.insecure_channel('localhost:50051') as channel: | ||
stub = imports_pb2_grpc.ImportsServiceStub(channel) | ||
try: | ||
|
||
hello_request = service_pb2.HelloRequest(action='hello with imports') | ||
request = imports_pb2.HelloImportsRequest(hello_request=hello_request) | ||
reply = stub.HelloImports(request) | ||
except grpc.RpcError as error: | ||
if error.code() == grpc.StatusCode.UNAVAILABLE: | ||
print("[ERROR] Connection to server is unavailable. You should create a server instance first.") | ||
print("To start a gRPC server, execute: `./pants run examples/src/python/example/grpcio/server`") | ||
else: | ||
print('An error occured! Error code: [{}] Error details: [{}]'.format(error.code(), error.details())) | ||
else: | ||
print(reply.hello_reply.response) | ||
print('[SUCCESS]') | ||
|
||
|
||
if __name__ == '__main__': | ||
run_example() |
31 changes: 31 additions & 0 deletions
31
examples/src/python/example/grpcio/client/service_client.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,31 @@ | ||
# coding=utf-8 | ||
# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). | ||
# Licensed under the Apache License, Version 2.0 (see LICENSE). | ||
|
||
from __future__ import absolute_import, division, print_function, unicode_literals | ||
|
||
import grpc | ||
from org.pantsbuild.example.grpcio.service import service_pb2, service_pb2_grpc | ||
|
||
|
||
def run_example(): | ||
print('hello world from grpcio service_client!') | ||
with grpc.insecure_channel('localhost:50051') as channel: | ||
stub = service_pb2_grpc.ExampleServiceStub(channel) | ||
try: | ||
hello_response = stub.Hello(service_pb2.HelloRequest(action='hello')) | ||
bye_response = stub.Hello(service_pb2.HelloRequest(action='bye')) | ||
except grpc.RpcError as error: | ||
if error.code() == grpc.StatusCode.UNAVAILABLE: | ||
print("[ERROR] Connection to server is unavailable. You should create a server instance first.") | ||
print("To start a gRPC server, execute: `./pants run examples/src/python/example/grpcio/server`") | ||
else: | ||
print('An error occured! Error code: [{}] Error details: [{}]'.format(error.code(), error.details())) | ||
else: | ||
print(hello_response) | ||
print(bye_response) | ||
print('[SUCCESS]') | ||
|
||
|
||
if __name__ == '__main__': | ||
run_example() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
# coding=utf-8 | ||
# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). | ||
# Licensed under the Apache License, Version 2.0 (see LICENSE). | ||
|
||
python_binary( | ||
name='server', | ||
dependencies=[ | ||
'3rdparty/python:grpcio', | ||
|
||
'examples/src/protobuf/org/pantsbuild/example/grpcio/service', | ||
'examples/src/protobuf/org/pantsbuild/example/grpcio/imports', | ||
], | ||
source='server.py', | ||
) |
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,52 @@ | ||
# coding=utf-8 | ||
# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). | ||
# Licensed under the Apache License, Version 2.0 (see LICENSE). | ||
|
||
from __future__ import absolute_import, division, print_function, unicode_literals | ||
|
||
import time | ||
from concurrent import futures | ||
|
||
import grpc | ||
from org.pantsbuild.example.grpcio.imports import imports_pb2, imports_pb2_grpc | ||
from org.pantsbuild.example.grpcio.service import service_pb2, service_pb2_grpc | ||
|
||
|
||
class ExampleHelloServer(service_pb2_grpc.ExampleServiceServicer): | ||
|
||
def Hello(self, request, context): | ||
print('request with action: [{}]'.format(request.action)) | ||
reply = service_pb2.HelloReply() | ||
reply.response = '{} from server!'.format(request.action) | ||
return reply | ||
|
||
|
||
class ImportsServiceServer(imports_pb2_grpc.ImportsServiceServicer): | ||
|
||
def HelloImports(self, request, context): | ||
print('request with action: [{}]'.format(request.hello_request.action)) | ||
hello_reply = service_pb2.HelloReply(response='{} from imports server!'.format(request.hello_request.action)) | ||
reply = imports_pb2.HelloImportsReply(hello_reply=hello_reply) | ||
return reply | ||
|
||
|
||
def run_server(): | ||
server = grpc.server(futures.ThreadPoolExecutor(max_workers=5)) | ||
|
||
service_pb2_grpc.add_ExampleServiceServicer_to_server(ExampleHelloServer(), server) | ||
imports_pb2_grpc.add_ImportsServiceServicer_to_server(ImportsServiceServer(), server) | ||
|
||
server.add_insecure_port('[::]:50051') | ||
server.start() | ||
|
||
print('Server is running...') | ||
print('(hit Ctrl+C to stop)') | ||
try: | ||
while True: | ||
time.sleep(10) | ||
except KeyboardInterrupt: | ||
server.stop(0) | ||
|
||
|
||
if __name__ == '__main__': | ||
run_server() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). | ||
# Licensed under the Apache License, Version 2.0 (see LICENSE). | ||
|
||
python_library( | ||
dependencies = [ | ||
'3rdparty/python:future', | ||
'3rdparty/python:pex', | ||
'src/python/pants/backend/python:plugin', | ||
'src/python/pants/backend/python/subsystems', | ||
'src/python/pants/backend/python/targets', | ||
'src/python/pants/base:workunit', | ||
'src/python/pants/base:build_environment', | ||
'src/python/pants/base:exceptions', | ||
'src/python/pants/build_graph', | ||
'src/python/pants/subsystem', | ||
'src/python/pants/util:contextutil', | ||
'src/python/pants/task', | ||
], | ||
) |
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
# coding=utf-8 | ||
# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). | ||
# Licensed under the Apache License, Version 2.0 (see LICENSE). | ||
|
||
from __future__ import absolute_import, division, print_function, unicode_literals | ||
|
||
from pants.backend.python.subsystems.python_tool_base import PythonToolBase | ||
|
||
|
||
class Grpcio(PythonToolBase): | ||
grpcio_version = '1.17.1' | ||
|
||
options_scope = 'grpcio' | ||
default_requirements = [ | ||
'grpcio-tools=={}'.format(grpcio_version), | ||
'grpcio=={}'.format(grpcio_version), | ||
] | ||
default_entry_point = 'grpc_tools.protoc' |
25 changes: 25 additions & 0 deletions
25
src/python/pants/backend/codegen/grpcio/python/grpcio_prep.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
# coding=utf-8 | ||
# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). | ||
# Licensed under the Apache License, Version 2.0 (see LICENSE). | ||
|
||
from __future__ import absolute_import, division, print_function, unicode_literals | ||
|
||
from pants.backend.codegen.grpcio.python.grpcio import Grpcio | ||
from pants.backend.codegen.grpcio.python.python_grpcio_library import PythonGrpcioLibrary | ||
from pants.backend.python.tasks.python_tool_prep_base import PythonToolInstance, PythonToolPrepBase | ||
|
||
|
||
class GrpcioInstance(PythonToolInstance): | ||
pass | ||
|
||
|
||
class GrpcioPrep(PythonToolPrepBase): | ||
tool_subsystem_cls = Grpcio | ||
tool_instance_cls = GrpcioInstance | ||
|
||
def execute(self): | ||
targets = self.get_targets(lambda target: isinstance(target, PythonGrpcioLibrary)) | ||
if not targets: | ||
return 0 | ||
|
||
super(GrpcioPrep, self).execute() |
61 changes: 61 additions & 0 deletions
61
src/python/pants/backend/codegen/grpcio/python/grpcio_run.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,61 @@ | ||
# coding=utf-8 | ||
# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). | ||
# Licensed under the Apache License, Version 2.0 (see LICENSE). | ||
|
||
from __future__ import absolute_import, division, print_function, unicode_literals | ||
|
||
import functools | ||
import logging | ||
|
||
from pants.backend.codegen.grpcio.python.grpcio_prep import GrpcioPrep | ||
from pants.backend.codegen.grpcio.python.python_grpcio_library import PythonGrpcioLibrary | ||
from pants.backend.python.targets.python_library import PythonLibrary | ||
from pants.base.build_environment import get_buildroot | ||
from pants.base.exceptions import TaskError | ||
from pants.base.workunit import WorkUnitLabel | ||
from pants.task.simple_codegen_task import SimpleCodegenTask | ||
from pants.util.contextutil import pushd | ||
from pants.util.memo import memoized_property | ||
|
||
|
||
class GrpcioRun(SimpleCodegenTask): | ||
"""Task to compile protobuf into python code""" | ||
|
||
gentarget_type = PythonGrpcioLibrary | ||
sources_globs = ('**/*',) | ||
|
||
@classmethod | ||
def prepare(cls, options, round_manager): | ||
super(GrpcioRun, cls).prepare(options, round_manager) | ||
round_manager.require_data(GrpcioPrep.tool_instance_cls) | ||
|
||
def synthetic_target_type(self, target): | ||
return PythonLibrary | ||
|
||
@memoized_property | ||
def _grpcio_binary(self): | ||
return self.context.products.get_data(GrpcioPrep.tool_instance_cls) | ||
|
||
def execute_codegen(self, target, target_workdir): | ||
args = self.build_args(target, target_workdir) | ||
logging.debug("Executing grpcio code generation with args: [{}]".format(args)) | ||
|
||
with pushd(get_buildroot()): | ||
workunit_factory = functools.partial(self.context.new_workunit, | ||
name='run-grpcio', | ||
labels=[WorkUnitLabel.TOOL, WorkUnitLabel.LINT]) | ||
cmdline, exit_code = self._grpcio_binary.run(workunit_factory, args) | ||
if exit_code != 0: | ||
raise TaskError('{} ... exited non-zero ({}).'.format(cmdline, exit_code), | ||
exit_code=exit_code) | ||
logging.info("Grpcio finished code generation into: [{}]".format(target_workdir)) | ||
|
||
def build_args(self, target, target_workdir): | ||
proto_path = '--proto_path={0}'.format(target.target_base) | ||
python_out = '--python_out={0}'.format(target_workdir) | ||
grpc_python_out = '--grpc_python_out={0}'.format(target_workdir) | ||
|
||
args = [python_out, grpc_python_out, proto_path] | ||
|
||
args.extend(target.sources_relative_to_buildroot()) | ||
return args |
14 changes: 14 additions & 0 deletions
14
src/python/pants/backend/codegen/grpcio/python/python_grpcio_library.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
# coding=utf-8 | ||
# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md). | ||
# Licensed under the Apache License, Version 2.0 (see LICENSE). | ||
|
||
from __future__ import absolute_import, division, print_function, unicode_literals | ||
|
||
from pants.backend.python.targets.python_target import PythonTarget | ||
|
||
|
||
class PythonGrpcioLibrary(PythonTarget): | ||
"""A Python library generated from Protocol Buffer IDL files.""" | ||
|
||
def __init__(self, sources=None, **kwargs): | ||
super(PythonGrpcioLibrary, self).__init__(sources=sources, **kwargs) |
Oops, something went wrong.