Protobuf stubs update using mypy-protobuf

.flake8

@@ -22,5 +22,5 @@ per-file-ignores =
 
 # We are checking with Python 3 but many of the stubs are Python 2 stubs.
 builtins = StandardError,apply,basestring,buffer,cmp,coerce,execfile,file,intern,long,raw_input,reduce,reload,unichr,unicode,xrange
-exclude = .venv*,@*,.git
+exclude = .venv*,@*,.git,*_pb2.pyi
 max-line-length = 130

pyproject.toml

@@ -1,11 +1,13 @@
 [tool.black]
 line_length = 130
 target_version = ["py37"]
+exclude = ".*_pb2.pyi"
 
 [tool.isort]
 profile = "black"
 combine_as_imports = true
 line_length = 130
+skip_glob = "*_pb2.pyi"
 extra_standard_library = [
     "typing_extensions",
     "_typeshed",

scripts/generate_proto_stubs.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+# Some of the proto .pyi stubs in third_party/2and3/google/protobuf/
+# are autogenerated using the mypy-protobuf project on the
+# latest `.proto` files shipped with protoc.
+#
+# When run, this script will autogenerate the _pb2.pyi stubs to
+# third_party/2and3/google/protobuf. It should be run any time there's
+# a meaningful update to either PROTOBUF_VERSION or MYPY_PROTOBUF_VERSION,
+# followed by committing the changes to typeshed
+#
+# Update these two variables when rerunning script
+PROTOBUF_VERSION=3.14.0
+MYPY_PROTOBUF_VERSION=v1.23
+
+set -ex
+
+if uname -a | grep Darwin; then
+    PLAT=osx
+else
+    PLAT=linux
+fi
+REPO_ROOT=$(realpath $(dirname "${BASH_SOURCE[0]}")/..)
+TMP_DIR=$(mktemp -d)
+PYTHON_PROTOBUF_FILENAME=protobuf-python-${PROTOBUF_VERSION}.zip
+PROTOC_FILENAME=protoc-${PROTOBUF_VERSION}-${PLAT}-x86_64.zip
+PROTOC_URL=https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/${PROTOC_FILENAME}
+PYTHON_PROTOBUF_URL=https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/${PYTHON_PROTOBUF_FILENAME}
+
+cd $TMP_DIR
+echo "Working in $TMP_DIR"
+
+# Install protoc
+wget $PROTOC_URL
+mkdir protoc_install
+unzip $PROTOC_FILENAME -d protoc_install
+
+# Fetch protoc-python (which contains all the .proto files)
+wget $PYTHON_PROTOBUF_URL
+unzip $PYTHON_PROTOBUF_FILENAME
+PYTHON_PROTOBUF_DIR=protobuf-$PROTOBUF_VERSION
+
+# Install mypy-protobuf
+VENV=venv
+python3 -m virtualenv $VENV
+source $VENV/bin/activate
+python3 -m pip install git+https://github.com/dropbox/mypy-protobuf@${MYPY_PROTOBUF_VERSION}#subdirectory=python
+
+# Remove existing pyi
+find $REPO_ROOT/third_party/2and3/ -name "*_pb2.pyi" -delete
+
+# Roughly reproduce the subset of .proto files on the public interface as described
+# by find_package_modules in the protobuf setup.py.
+# The logic (as of 3.14.0) can roughly be described as a whitelist of .proto files
+# further limited to exclude *test* and internal/
+# https://github.com/protocolbuffers/protobuf/blob/master/python/setup.py
+PROTO_FILES=$(grep "generate_proto.*google" $PYTHON_PROTOBUF_DIR/python/setup.py | \
+    cut -d\" -f2 | \
+    grep -v "test" | \
+    grep -v google/protobuf/internal/ | \
+    grep -v google/protobuf/pyext/python.proto | \
+    sed "s:^:$PYTHON_PROTOBUF_DIR/python/:" | \
+    xargs -L1 realpath --relative-to=. \
+)
+
+# And regenerate!
+protoc_install/bin/protoc --proto_path=$PYTHON_PROTOBUF_DIR/src --mypy_out=$REPO_ROOT/third_party/2and3 $PROTO_FILES

tests/pytype_exclude_list.txt

@@ -16,3 +16,31 @@ third_party/2and3/attr/converters.pyi
 third_party/2and3/attr/filters.pyi
 third_party/2and3/attr/validators.pyi
 third_party/2and3/pynamodb/models.pyi
+
+# _pb2.pyi have some constructs that break pytype
+# Eg
+# pytype.pyi.parser.ParseError:   File: "/Users/nipunn/src/typeshed/third_party/2and3/google/protobuf/descriptor_pb2.pyi", line 195
+# b"TypeValue = typing___NewType('TypeValue', builtin___int)"
+third_party/2and3/google/protobuf/any_pb2.pyi
+third_party/2and3/google/protobuf/api_pb2.pyi
+third_party/2and3/google/protobuf/compiler/plugin_pb2.pyi
+third_party/2and3/google/protobuf/descriptor.pyi
+third_party/2and3/google/protobuf/descriptor_pb2.pyi
+third_party/2and3/google/protobuf/duration_pb2.pyi
+third_party/2and3/google/protobuf/empty_pb2.pyi
+third_party/2and3/google/protobuf/field_mask_pb2.pyi
+third_party/2and3/google/protobuf/internal/containers.pyi
+third_party/2and3/google/protobuf/internal/enum_type_wrapper.pyi
+third_party/2and3/google/protobuf/internal/extension_dict.pyi
+third_party/2and3/google/protobuf/json_format.pyi
+third_party/2and3/google/protobuf/message.pyi
+third_party/2and3/google/protobuf/message_factory.pyi
+third_party/2and3/google/protobuf/service.pyi
+third_party/2and3/google/protobuf/source_context_pb2.pyi
+third_party/2and3/google/protobuf/struct_pb2.pyi
+third_party/2and3/google/protobuf/symbol_database.pyi
+third_party/2and3/google/protobuf/timestamp_pb2.pyi
+third_party/2and3/google/protobuf/type_pb2.pyi
+third_party/2and3/google/protobuf/util/json_format_pb2.pyi
+third_party/2and3/google/protobuf/util/json_format_proto3_pb2.pyi
+third_party/2and3/google/protobuf/wrappers_pb2.pyi

third_party/2and3/google/protobuf/any_pb2.pyi

@@ -1,9 +1,41 @@
-from typing import Optional, Text
+# @generated by generate_proto_mypy_stubs.py.  Do not edit!
+import sys
+from google.protobuf.descriptor import (
+    Descriptor as google___protobuf___descriptor___Descriptor,
+    FileDescriptor as google___protobuf___descriptor___FileDescriptor,
+)
 
-from google.protobuf.internal import well_known_types
-from google.protobuf.message import Message
+from google.protobuf.message import (
+    Message as google___protobuf___message___Message,
+)
 
-class Any(Message, well_known_types.Any_):
-    type_url: Text
-    value: bytes
-    def __init__(self, type_url: Optional[Text] = ..., value: Optional[bytes] = ...) -> None: ...
+from typing import (
+    Optional as typing___Optional,
+    Text as typing___Text,
+)
+
+from typing_extensions import (
+    Literal as typing_extensions___Literal,
+)
+
+
+builtin___bool = bool
+builtin___bytes = bytes
+builtin___float = float
+builtin___int = int
+
+
+DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
+
+class Any(google___protobuf___message___Message):
+    DESCRIPTOR: google___protobuf___descriptor___Descriptor = ...
+    type_url: typing___Text = ...
+    value: builtin___bytes = ...
+
+    def __init__(self,
+        *,
+        type_url : typing___Optional[typing___Text] = None,
+        value : typing___Optional[builtin___bytes] = None,
+        ) -> None: ...
+    def ClearField(self, field_name: typing_extensions___Literal[u"type_url",b"type_url",u"value",b"value"]) -> None: ...
+type___Any = Any

third_party/2and3/google/protobuf/api_pb2.pyi

@@ -1,54 +1,112 @@
-from typing import Iterable, Optional, Text
+# @generated by generate_proto_mypy_stubs.py.  Do not edit!
+import sys
+from google.protobuf.descriptor import (
+    Descriptor as google___protobuf___descriptor___Descriptor,
+    FileDescriptor as google___protobuf___descriptor___FileDescriptor,
+)
 
-from google.protobuf.internal.containers import RepeatedCompositeFieldContainer
-from google.protobuf.message import Message
-from google.protobuf.source_context_pb2 import SourceContext
-from google.protobuf.type_pb2 import Option, Syntax
+from google.protobuf.internal.containers import (
+    RepeatedCompositeFieldContainer as google___protobuf___internal___containers___RepeatedCompositeFieldContainer,
+)
+
+from google.protobuf.message import (
+    Message as google___protobuf___message___Message,
+)
+
+from google.protobuf.source_context_pb2 import (
+    SourceContext as google___protobuf___source_context_pb2___SourceContext,
+)
+
+from google.protobuf.type_pb2 import (
+    Option as google___protobuf___type_pb2___Option,
+    SyntaxValue as google___protobuf___type_pb2___SyntaxValue,
+)
+
+from typing import (
+    Iterable as typing___Iterable,
+    Optional as typing___Optional,
+    Text as typing___Text,
+)
+
+from typing_extensions import (
+    Literal as typing_extensions___Literal,
+)
+
+
+builtin___bool = bool
+builtin___bytes = bytes
+builtin___float = float
+builtin___int = int
+
+
+DESCRIPTOR: google___protobuf___descriptor___FileDescriptor = ...
+
+class Api(google___protobuf___message___Message):
+    DESCRIPTOR: google___protobuf___descriptor___Descriptor = ...
+    name: typing___Text = ...
+    version: typing___Text = ...
+    syntax: google___protobuf___type_pb2___SyntaxValue = ...
 
-class Api(Message):
-    name: Text
-    version: Text
-    syntax: Syntax
     @property
-    def methods(self) -> RepeatedCompositeFieldContainer[Method]: ...
+    def methods(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[type___Method]: ...
+
     @property
-    def options(self) -> RepeatedCompositeFieldContainer[Option]: ...
+    def options(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[google___protobuf___type_pb2___Option]: ...
+
     @property
-    def source_context(self) -> SourceContext: ...
+    def source_context(self) -> google___protobuf___source_context_pb2___SourceContext: ...
+
     @property
-    def mixins(self) -> RepeatedCompositeFieldContainer[Mixin]: ...
-    def __init__(
-        self,
-        name: Optional[Text] = ...,
-        methods: Optional[Iterable[Method]] = ...,
-        options: Optional[Iterable[Option]] = ...,
-        version: Optional[Text] = ...,
-        source_context: Optional[SourceContext] = ...,
-        mixins: Optional[Iterable[Mixin]] = ...,
-        syntax: Optional[Syntax] = ...,
-    ) -> None: ...
-
-class Method(Message):
-    name: Text
-    request_type_url: Text
-    request_streaming: bool
-    response_type_url: Text
-    response_streaming: bool
-    syntax: Syntax
+    def mixins(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[type___Mixin]: ...
+
+    def __init__(self,
+        *,
+        name : typing___Optional[typing___Text] = None,
+        methods : typing___Optional[typing___Iterable[type___Method]] = None,
+        options : typing___Optional[typing___Iterable[google___protobuf___type_pb2___Option]] = None,
+        version : typing___Optional[typing___Text] = None,
+        source_context : typing___Optional[google___protobuf___source_context_pb2___SourceContext] = None,
+        mixins : typing___Optional[typing___Iterable[type___Mixin]] = None,
+        syntax : typing___Optional[google___protobuf___type_pb2___SyntaxValue] = None,
+        ) -> None: ...
+    def HasField(self, field_name: typing_extensions___Literal[u"source_context",b"source_context"]) -> builtin___bool: ...
+    def ClearField(self, field_name: typing_extensions___Literal[u"methods",b"methods",u"mixins",b"mixins",u"name",b"name",u"options",b"options",u"source_context",b"source_context",u"syntax",b"syntax",u"version",b"version"]) -> None: ...
+type___Api = Api
+
+class Method(google___protobuf___message___Message):
+    DESCRIPTOR: google___protobuf___descriptor___Descriptor = ...
+    name: typing___Text = ...
+    request_type_url: typing___Text = ...
+    request_streaming: builtin___bool = ...
+    response_type_url: typing___Text = ...
+    response_streaming: builtin___bool = ...
+    syntax: google___protobuf___type_pb2___SyntaxValue = ...
+
     @property
-    def options(self) -> RepeatedCompositeFieldContainer[Option]: ...
-    def __init__(
-        self,
-        name: Optional[Text] = ...,
-        request_type_url: Optional[Text] = ...,
-        request_streaming: Optional[bool] = ...,
-        response_type_url: Optional[Text] = ...,
-        response_streaming: Optional[bool] = ...,
-        options: Optional[Iterable[Option]] = ...,
-        syntax: Optional[Syntax] = ...,
-    ) -> None: ...
-
-class Mixin(Message):
-    name: Text
-    root: Text
-    def __init__(self, name: Optional[Text] = ..., root: Optional[Text] = ...) -> None: ...
+    def options(self) -> google___protobuf___internal___containers___RepeatedCompositeFieldContainer[google___protobuf___type_pb2___Option]: ...
+
+    def __init__(self,
+        *,
+        name : typing___Optional[typing___Text] = None,
+        request_type_url : typing___Optional[typing___Text] = None,
+        request_streaming : typing___Optional[builtin___bool] = None,
+        response_type_url : typing___Optional[typing___Text] = None,
+        response_streaming : typing___Optional[builtin___bool] = None,
+        options : typing___Optional[typing___Iterable[google___protobuf___type_pb2___Option]] = None,
+        syntax : typing___Optional[google___protobuf___type_pb2___SyntaxValue] = None,
+        ) -> None: ...
+    def ClearField(self, field_name: typing_extensions___Literal[u"name",b"name",u"options",b"options",u"request_streaming",b"request_streaming",u"request_type_url",b"request_type_url",u"response_streaming",b"response_streaming",u"response_type_url",b"response_type_url",u"syntax",b"syntax"]) -> None: ...
+type___Method = Method
+
+class Mixin(google___protobuf___message___Message):
+    DESCRIPTOR: google___protobuf___descriptor___Descriptor = ...
+    name: typing___Text = ...
+    root: typing___Text = ...
+
+    def __init__(self,
+        *,
+        name : typing___Optional[typing___Text] = None,
+        root : typing___Optional[typing___Text] = None,
+        ) -> None: ...
+    def ClearField(self, field_name: typing_extensions___Literal[u"name",b"name",u"root",b"root"]) -> None: ...
+type___Mixin = Mixin