Divide Wheels WIP local PR

ci/scripts/python_wheel_manylinux_build.sh

@@ -160,8 +160,8 @@ export ARROW_HOME=/tmp/arrow-dist
 export CMAKE_PREFIX_PATH=/tmp/arrow-dist
 
 pushd /arrow/python
-python setup.py bdist_wheel
-
+python setup.py bdist_wheel --target pyarrow_core
+# --exclude libarrow_flight.so --exclude libarrow_python_flight.so
 echo "=== (${PYTHON_VERSION}) Tag the wheel with manylinux${MANYLINUX_VERSION} ==="
-auditwheel repair -L . dist/pyarrow-*.whl -w repaired_wheels
+#auditwheel repair -L . dist/pyarrow*.whl -w repaired_wheels
 popd

python/pyarrow/flight.py

@@ -14,51 +14,54 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-
-from pyarrow._flight import (  # noqa:F401
-    connect,
-    Action,
-    ActionType,
-    BasicAuth,
-    CallInfo,
-    CertKeyPair,
-    ClientAuthHandler,
-    ClientMiddleware,
-    ClientMiddlewareFactory,
-    DescriptorType,
-    FlightCallOptions,
-    FlightCancelledError,
-    FlightClient,
-    FlightDataStream,
-    FlightDescriptor,
-    FlightEndpoint,
-    FlightError,
-    FlightInfo,
-    FlightInternalError,
-    FlightMetadataReader,
-    FlightMetadataWriter,
-    FlightMethod,
-    FlightServerBase,
-    FlightServerError,
-    FlightStreamChunk,
-    FlightStreamReader,
-    FlightStreamWriter,
-    FlightTimedOutError,
-    FlightUnauthenticatedError,
-    FlightUnauthorizedError,
-    FlightUnavailableError,
-    FlightWriteSizeExceededError,
-    GeneratorStream,
-    Location,
-    MetadataRecordBatchReader,
-    MetadataRecordBatchWriter,
-    RecordBatchStream,
-    Result,
-    SchemaResult,
-    ServerAuthHandler,
-    ServerCallContext,
-    ServerMiddleware,
-    ServerMiddlewareFactory,
-    Ticket,
-    TracingServerMiddlewareFactory,
-)
+try:
+    from pyarrow._flight import (  # noqa:F401
+        connect,
+        Action,
+        ActionType,
+        BasicAuth,
+        CallInfo,
+        CertKeyPair,
+        ClientAuthHandler,
+        ClientMiddleware,
+        ClientMiddlewareFactory,
+        DescriptorType,
+        FlightCallOptions,
+        FlightCancelledError,
+        FlightClient,
+        FlightDataStream,
+        FlightDescriptor,
+        FlightEndpoint,
+        FlightError,
+        FlightInfo,
+        FlightInternalError,
+        FlightMetadataReader,
+        FlightMetadataWriter,
+        FlightMethod,
+        FlightServerBase,
+        FlightServerError,
+        FlightStreamChunk,
+        FlightStreamReader,
+        FlightStreamWriter,
+        FlightTimedOutError,
+        FlightUnauthenticatedError,
+        FlightUnauthorizedError,
+        FlightUnavailableError,
+        FlightWriteSizeExceededError,
+        GeneratorStream,
+        Location,
+        MetadataRecordBatchReader,
+        MetadataRecordBatchWriter,
+        RecordBatchStream,
+        Result,
+        SchemaResult,
+        ServerAuthHandler,
+        ServerCallContext,
+        ServerMiddleware,
+        ServerMiddlewareFactory,
+        Ticket,
+        TracingServerMiddlewareFactory,
+    )
+except ImportError:
+    raise ImportError('Flight extension not installed. You can install it '
+                    'with pip install pyarrow_flight=={version} or conda install pyarrow_flight=={version}.')

python/setup.py

@@ -47,6 +47,55 @@
 
 ext_suffix = sysconfig.get_config_var('EXT_SUFFIX')
 
+import argparse
+
+# Pre-parse and remove the '--target' argument from sys.argv
+parser = argparse.ArgumentParser()
+parser.add_argument('--target', default='pyarrow')
+values, rest = parser.parse_known_args()
+DIST_TARGET = values.target
+sys.argv = sys.argv[:1] + rest
+
+class Target():
+    def __init__(self, target):
+        if target not in ['pyarrow', 'pyarrow_core', 'pyarrow_flight']:
+            raise ValueError("unknown distribution target: '%s'" % target)
+        self.target = target
+
+    def get_packages(self):
+        if strtobool(os.environ.get('PYARROW_INSTALL_TESTS', '1')):
+            return ['pyarrow', 'pyarrow.tests']
+        else:
+            return ['pyarrow']
+
+    def get_package_data(self):
+        return {'pyarrow': ['*.pxd', '*.pyx', 'includes/*.pxd']}
+
+    def get_description(self):
+        if self.target == 'pyarrow':
+            return 'Python library for Apache Arrow'
+        elif self.target == 'pyarrow_core':
+            return 'Python library for core Apache Arrow features'
+        elif self.target == 'pyarrow_flight':
+            return 'Python library for core Apache Arrow features'
+
+    def get_long_description(self):
+        # TODO
+        if self.target == 'pyarrow':
+            with open('README.md') as f:
+                long_description = f.read()
+        else:
+            with open(os.path.join(self.target, 'README.md')) as f:
+                long_description = f.read()
+        return long_description
+
+    def get_test_suite(self):
+        # TODO
+        return '%s.tests' % self.target
+
+
+target = Target(DIST_TARGET)
+
 
 @contextlib.contextmanager
 def changed_dir(dirname):
@@ -83,6 +132,7 @@ def build_extensions(self):
 
         self.extensions = [ext for ext in self.extensions
                            if ext.name != '__dummy__']
+        breakpoint()
 
         for ext in self.extensions:
             if (hasattr(ext, 'include_dirs') and
@@ -472,10 +522,10 @@ def has_ext_modules(foo):
 
 
 setup(
-    name='pyarrow',
-    packages=packages,
+    name=target.target,
+    packages=target.get_packages(),
     zip_safe=False,
-    package_data={'pyarrow': ['*.pxd', '*.pyx', 'includes/*.pxd']},
+    package_data=target.get_package_data(),
     include_package_data=True,
     exclude_package_data=exclude_package_data,
     distclass=BinaryDistribution,