Fix python package and publish typing information (#347)

* Add __init__.py files to generated stubs Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> * Publish stubs in the package Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> * Include __init__.py in verification workflow Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> * Bump versions of remote plugins Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com> Co-authored-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
flyteorg · Nov 30, 2022 · b4a6a57 · b4a6a57
1 parent f4ede35
commit b4a6a57
Show file tree

Hide file tree

Showing 13 changed files with 15 additions and 2 deletions.
diff --git a/.github/workflows/verification.yml b/.github/workflows/verification.yml
@@ -113,6 +113,8 @@ jobs:
         run: rm -rf ./gen
       - name: "Run buf generate"
         run: buf generate
+      - name: "Include __init__.py files"
+        run: find gen/pb_python -type d -exec touch {}/__init__.py \;
       - uses: actions/upload-artifact@master
         with:
           name: generated-buf

diff --git a/buf.gen.yaml b/buf.gen.yaml
@@ -2,9 +2,9 @@ version: v1
 managed:
   enabled: true
 plugins:
-  - remote: buf.build/grpc/plugins/python:v1.49.1-1
+  - remote: buf.build/grpc/plugins/python:v1.51.1-1
     out: gen/pb_python
-  - remote: buf.build/protocolbuffers/plugins/python:v21.7.0-1
+  - remote: buf.build/protocolbuffers/plugins/python:v21.9.0-1
     out: gen/pb_python
   - remote: buf.build/protocolbuffers/plugins/pyi:v21.9.0-1
     out: gen/pb_python
diff --git a/gen/pb_python/__init__.py b/gen/pb_python/__init__.py
diff --git a/gen/pb_python/flyteidl/__init__.py b/gen/pb_python/flyteidl/__init__.py
diff --git a/gen/pb_python/flyteidl/admin/__init__.py b/gen/pb_python/flyteidl/admin/__init__.py
diff --git a/gen/pb_python/flyteidl/core/__init__.py b/gen/pb_python/flyteidl/core/__init__.py
diff --git a/gen/pb_python/flyteidl/datacatalog/__init__.py b/gen/pb_python/flyteidl/datacatalog/__init__.py
diff --git a/gen/pb_python/flyteidl/event/__init__.py b/gen/pb_python/flyteidl/event/__init__.py
diff --git a/gen/pb_python/flyteidl/plugins/__init__.py b/gen/pb_python/flyteidl/plugins/__init__.py
diff --git a/gen/pb_python/flyteidl/plugins/sagemaker/__init__.py b/gen/pb_python/flyteidl/plugins/sagemaker/__init__.py
diff --git a/gen/pb_python/flyteidl/service/__init__.py b/gen/pb_python/flyteidl/service/__init__.py
diff --git a/generate_protos.sh b/generate_protos.sh
@@ -29,9 +29,16 @@ do
 done
 
 # Buf migration
+# TODO: fix stub generation via docker image. The trick of forcing the current user as a user in
+# the container (used extensively in this file) does not work.
 # docker run --volume "$(pwd):/workspace" --workdir /workspace bufbuild/buf generate
 buf generate
 
+# Unfortunately the python protoc plugin does not add __init__.py files to the generated code
+# (as described in https://github.com/protocolbuffers/protobuf/issues/881). One of the
+# suggestions is to manually create such files, which is what we do here:
+find gen/pb_python -type d -exec touch {}/__init__.py \;
+
 # Docs generated
 
 # Remove any currently generated core docs file

diff --git a/setup.py b/setup.py
@@ -11,6 +11,10 @@
     maintainer_email='admin@flyte.org',
     packages=find_packages('gen/pb_python'),
     package_dir={'': 'gen/pb_python'},
+    # https://github.com/pypa/setuptools/issues/3136 describes an extension to
+    # setuptools that would involve a simpler way to specify this, but while
+    # that does not happen we have to package the pyi files manually like so:
+    package_data={'flyteidl': ["*.pyi", "**/*.pyi"]},
     dependency_links=[],
     install_requires=[
         'googleapis-common-protos',