Python code generation for protobufs and gRPC (pantsbuild#6974)

### Problem

I would like to have a task, to generate python code from protobufs and grpc services.

### Solution

There is a new codegen task **grpcio-run**, to execute python's grpcio library <https://grpc.io/> and generate python code from .proto files.

### Example:

Respectful examples can be found in [/examples/src/python/example/grpcio](/examples/src/python/example/grpcio)

to create a gRPC server execute 
```./pants run examples/src/python/example/grpcio/server```

and when it's running, run client example:
```./pants run examples/src/python/example/grpcio/client```

generated code can be found as usual in pants output directory:
```/.pants.d/gen/grpcio-run/current/examples.src.protobuf.org.pantsbuild.example.service.service/current/org/pantsbuild/example/service```

3rdparty/python/requirements.txt

@@ -10,13 +10,15 @@ fasteners==0.14.1
 faulthandler==2.6 ; python_version<'3'
 future==0.16.0
 futures==3.0.5 ; python_version<'3'
+grpcio==1.16.1
 Markdown==2.1.1
 mock==2.0.0
 packaging==16.8
 parameterized==0.6.1
 pathspec==0.5.9
 pex==1.5.3
 psutil==4.3.0
+protobuf==3.6.1
 pycodestyle==2.4.0
 pyflakes==2.0.0
 Pygments==2.3.1

examples/src/protobuf/org/pantsbuild/example/grpcio/imports/BUILD

@@ -0,0 +1,11 @@
+# coding=utf-8
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_grpcio_library(
+  sources=['imports.proto'],
+  dependencies=[
+    '3rdparty/python:protobuf',
+    'examples/src/protobuf/org/pantsbuild/example/grpcio/service'
+  ]
+)

examples/src/protobuf/org/pantsbuild/example/grpcio/imports/imports.proto

@@ -0,0 +1,16 @@
+syntax = "proto3";
+package org.pantsbuild.example.grpcio.imports;
+
+import "org/pantsbuild/example/grpcio/service/service.proto";
+
+service ImportsService {
+    rpc HelloImports(HelloImportsRequest) returns (HelloImportsReply) {}
+}
+
+message HelloImportsRequest {
+    org.pantsbuild.example.grpcio.service.HelloRequest hello_request = 1;
+}
+
+message HelloImportsReply{
+    org.pantsbuild.example.grpcio.service.HelloReply hello_reply = 1;
+}

examples/src/protobuf/org/pantsbuild/example/grpcio/service/BUILD

@@ -0,0 +1,10 @@
+# coding=utf-8
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_grpcio_library(
+  sources=['service.proto'],
+  dependencies=[
+    '3rdparty/python:protobuf',
+  ]
+)

examples/src/protobuf/org/pantsbuild/example/grpcio/service/service.proto

@@ -0,0 +1,12 @@
+syntax = "proto3";
+package org.pantsbuild.example.grpcio.service;
+
+service ExampleService {
+    rpc Hello(HelloRequest) returns (HelloReply) {}
+}
+message HelloRequest {
+    string action = 1;
+}
+message HelloReply {
+    string response = 1;
+}

examples/src/python/example/grpcio/client/BUILD

@@ -0,0 +1,22 @@
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_binary(
+  name='service',
+  dependencies=[
+    '3rdparty/python:grpcio',
+
+    'examples/src/protobuf/org/pantsbuild/example/grpcio/service',
+  ],
+  source='service_client.py',
+)
+
+python_binary(
+  name='imports',
+  dependencies=[
+    '3rdparty/python:grpcio',
+
+    'examples/src/protobuf/org/pantsbuild/example/grpcio/imports',
+  ],
+  source='imports_client.py',
+)

examples/src/python/example/grpcio/client/imports_client.py

@@ -0,0 +1,33 @@
+# coding=utf-8
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import grpc
+from org.pantsbuild.example.grpcio.imports import imports_pb2, imports_pb2_grpc
+from org.pantsbuild.example.grpcio.service import service_pb2
+
+
+def run_example():
+  print('hello world from grpcio imports_client!')
+  with grpc.insecure_channel('localhost:50051') as channel:
+    stub = imports_pb2_grpc.ImportsServiceStub(channel)
+    try:
+
+      hello_request = service_pb2.HelloRequest(action='hello with imports')
+      request = imports_pb2.HelloImportsRequest(hello_request=hello_request)
+      reply = stub.HelloImports(request)
+    except grpc.RpcError as error:
+      if error.code() == grpc.StatusCode.UNAVAILABLE:
+        print("[ERROR] Connection to server is unavailable. You should create a server instance first.")
+        print("To start a gRPC server, execute: `./pants run examples/src/python/example/grpcio/server`")
+      else:
+        print('An error occured! Error code: [{}] Error details: [{}]'.format(error.code(), error.details()))
+    else:
+      print(reply.hello_reply.response)
+      print('[SUCCESS]')
+
+
+if __name__ == '__main__':
+  run_example()

examples/src/python/example/grpcio/client/service_client.py

@@ -0,0 +1,31 @@
+# coding=utf-8
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import grpc
+from org.pantsbuild.example.grpcio.service import service_pb2, service_pb2_grpc
+
+
+def run_example():
+  print('hello world from grpcio service_client!')
+  with grpc.insecure_channel('localhost:50051') as channel:
+    stub = service_pb2_grpc.ExampleServiceStub(channel)
+    try:
+      hello_response = stub.Hello(service_pb2.HelloRequest(action='hello'))
+      bye_response = stub.Hello(service_pb2.HelloRequest(action='bye'))
+    except grpc.RpcError as error:
+      if error.code() == grpc.StatusCode.UNAVAILABLE:
+        print("[ERROR] Connection to server is unavailable. You should create a server instance first.")
+        print("To start a gRPC server, execute: `./pants run examples/src/python/example/grpcio/server`")
+      else:
+        print('An error occured! Error code: [{}] Error details: [{}]'.format(error.code(), error.details()))
+    else:
+      print(hello_response)
+      print(bye_response)
+      print('[SUCCESS]')
+
+
+if __name__ == '__main__':
+  run_example()

examples/src/python/example/grpcio/server/BUILD

@@ -0,0 +1,14 @@
+# coding=utf-8
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_binary(
+  name='server',
+  dependencies=[
+    '3rdparty/python:grpcio',
+
+    'examples/src/protobuf/org/pantsbuild/example/grpcio/service',
+    'examples/src/protobuf/org/pantsbuild/example/grpcio/imports',
+  ],
+  source='server.py',
+)

examples/src/python/example/grpcio/server/server.py

@@ -0,0 +1,52 @@
+# coding=utf-8
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import time
+from concurrent import futures
+
+import grpc
+from org.pantsbuild.example.grpcio.imports import imports_pb2, imports_pb2_grpc
+from org.pantsbuild.example.grpcio.service import service_pb2, service_pb2_grpc
+
+
+class ExampleHelloServer(service_pb2_grpc.ExampleServiceServicer):
+
+  def Hello(self, request, context):
+    print('request with action: [{}]'.format(request.action))
+    reply = service_pb2.HelloReply()
+    reply.response = '{} from server!'.format(request.action)
+    return reply
+
+
+class ImportsServiceServer(imports_pb2_grpc.ImportsServiceServicer):
+
+  def HelloImports(self, request, context):
+    print('request with action: [{}]'.format(request.hello_request.action))
+    hello_reply = service_pb2.HelloReply(response='{} from imports server!'.format(request.hello_request.action))
+    reply = imports_pb2.HelloImportsReply(hello_reply=hello_reply)
+    return reply
+
+
+def run_server():
+  server = grpc.server(futures.ThreadPoolExecutor(max_workers=5))
+
+  service_pb2_grpc.add_ExampleServiceServicer_to_server(ExampleHelloServer(), server)
+  imports_pb2_grpc.add_ImportsServiceServicer_to_server(ImportsServiceServer(), server)
+
+  server.add_insecure_port('[::]:50051')
+  server.start()
+
+  print('Server is running...')
+  print('(hit Ctrl+C to stop)')
+  try:
+    while True:
+      time.sleep(10)
+  except KeyboardInterrupt:
+    server.stop(0)
+
+
+if __name__ == '__main__':
+  run_server()

src/python/pants/backend/codegen/BUILD

@@ -13,6 +13,7 @@ python_library(
     'src/python/pants/backend/codegen/thrift/java',
     'src/python/pants/backend/codegen/thrift/python',
     'src/python/pants/backend/codegen/wire/java',
+    'src/python/pants/backend/codegen/grpcio/python',
     'src/python/pants/build_graph',
     'src/python/pants/goal:task_registrar',
   ]

src/python/pants/backend/codegen/grpcio/python/BUILD

@@ -0,0 +1,19 @@
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+python_library(
+  dependencies = [
+    '3rdparty/python:future',
+    '3rdparty/python:pex',
+    'src/python/pants/backend/python:plugin',
+    'src/python/pants/backend/python/subsystems',
+    'src/python/pants/backend/python/targets',
+    'src/python/pants/base:workunit',
+    'src/python/pants/base:build_environment',
+    'src/python/pants/base:exceptions',
+    'src/python/pants/build_graph',
+    'src/python/pants/subsystem',
+    'src/python/pants/util:contextutil',
+    'src/python/pants/task',
+  ],
+)

src/python/pants/backend/codegen/grpcio/python/grpcio.py

@@ -0,0 +1,18 @@
+# coding=utf-8
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from pants.backend.python.subsystems.python_tool_base import PythonToolBase
+
+
+class Grpcio(PythonToolBase):
+  grpcio_version = '1.17.1'
+
+  options_scope = 'grpcio'
+  default_requirements = [
+    'grpcio-tools=={}'.format(grpcio_version),
+    'grpcio=={}'.format(grpcio_version),
+  ]
+  default_entry_point = 'grpc_tools.protoc'

src/python/pants/backend/codegen/grpcio/python/grpcio_prep.py

@@ -0,0 +1,25 @@
+# coding=utf-8
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from pants.backend.codegen.grpcio.python.grpcio import Grpcio
+from pants.backend.codegen.grpcio.python.python_grpcio_library import PythonGrpcioLibrary
+from pants.backend.python.tasks.python_tool_prep_base import PythonToolInstance, PythonToolPrepBase
+
+
+class GrpcioInstance(PythonToolInstance):
+  pass
+
+
+class GrpcioPrep(PythonToolPrepBase):
+  tool_subsystem_cls = Grpcio
+  tool_instance_cls = GrpcioInstance
+
+  def execute(self):
+    targets = self.get_targets(lambda target: isinstance(target, PythonGrpcioLibrary))
+    if not targets:
+      return 0
+
+    super(GrpcioPrep, self).execute()

src/python/pants/backend/codegen/grpcio/python/grpcio_run.py

@@ -0,0 +1,61 @@
+# coding=utf-8
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import functools
+import logging
+
+from pants.backend.codegen.grpcio.python.grpcio_prep import GrpcioPrep
+from pants.backend.codegen.grpcio.python.python_grpcio_library import PythonGrpcioLibrary
+from pants.backend.python.targets.python_library import PythonLibrary
+from pants.base.build_environment import get_buildroot
+from pants.base.exceptions import TaskError
+from pants.base.workunit import WorkUnitLabel
+from pants.task.simple_codegen_task import SimpleCodegenTask
+from pants.util.contextutil import pushd
+from pants.util.memo import memoized_property
+
+
+class GrpcioRun(SimpleCodegenTask):
+  """Task to compile protobuf into python code"""
+
+  gentarget_type = PythonGrpcioLibrary
+  sources_globs = ('**/*',)
+
+  @classmethod
+  def prepare(cls, options, round_manager):
+    super(GrpcioRun, cls).prepare(options, round_manager)
+    round_manager.require_data(GrpcioPrep.tool_instance_cls)
+
+  def synthetic_target_type(self, target):
+    return PythonLibrary
+
+  @memoized_property
+  def _grpcio_binary(self):
+    return self.context.products.get_data(GrpcioPrep.tool_instance_cls)
+
+  def execute_codegen(self, target, target_workdir):
+    args = self.build_args(target, target_workdir)
+    logging.debug("Executing grpcio code generation with args: [{}]".format(args))
+
+    with pushd(get_buildroot()):
+      workunit_factory = functools.partial(self.context.new_workunit,
+                                           name='run-grpcio',
+                                           labels=[WorkUnitLabel.TOOL, WorkUnitLabel.LINT])
+      cmdline, exit_code = self._grpcio_binary.run(workunit_factory, args)
+      if exit_code != 0:
+        raise TaskError('{} ... exited non-zero ({}).'.format(cmdline, exit_code),
+                        exit_code=exit_code)
+      logging.info("Grpcio finished code generation into: [{}]".format(target_workdir))
+
+  def build_args(self, target, target_workdir):
+    proto_path = '--proto_path={0}'.format(target.target_base)
+    python_out = '--python_out={0}'.format(target_workdir)
+    grpc_python_out = '--grpc_python_out={0}'.format(target_workdir)
+
+    args = [python_out, grpc_python_out, proto_path]
+
+    args.extend(target.sources_relative_to_buildroot())
+    return args

src/python/pants/backend/codegen/grpcio/python/python_grpcio_library.py

@@ -0,0 +1,14 @@
+# coding=utf-8
+# Copyright 2019 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from pants.backend.python.targets.python_target import PythonTarget
+
+
+class PythonGrpcioLibrary(PythonTarget):
+  """A Python library generated from Protocol Buffer IDL files."""
+
+  def __init__(self, sources=None, **kwargs):
+    super(PythonGrpcioLibrary, self).__init__(sources=sources, **kwargs)