From cab55cb250bdec3063e538a2abde80c6257b3c42 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 1 Nov 2016 17:10:27 -0400 Subject: [PATCH 1/4] Opt in to building the pyarrow.parquet extension, do not silently fail Change-Id: Ie713bccbc89fc671ffd7f32bf2fb0f89b1c6d646 --- python/CMakeLists.txt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 179f02fbc9daa..6ad55f8c9a7b8 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -50,6 +50,9 @@ if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") option(PYARROW_BUILD_TESTS "Build the PyArrow C++ googletest unit tests" OFF) + option(PYARROW_BUILD_PARQUET + "Build the PyArrow Parquet integration" + OFF) endif() find_program(CCACHE_FOUND ccache) @@ -445,7 +448,10 @@ set(LINK_LIBS arrow_ipc ) -if(PARQUET_FOUND AND PARQUET_ARROW_FOUND) +if (PYARROW_BUILD_PARQUET) + if(NOT (PARQUET_FOUND AND PARQUET_ARROW_FOUND)) + message(FATAL_ERROR "Unable to locate Parquet libraries") + endif() ADD_THIRDPARTY_LIB(parquet_arrow SHARED_LIB ${PARQUET_ARROW_SHARED_LIB}) set(LINK_LIBS From 374e254e3d26eb5ddbf913e0c08f82e7eed2294b Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 1 Nov 2016 17:12:02 -0400 Subject: [PATCH 2/4] Add to README about building the parquet extension Change-Id: I32fe58a984d32044d9313cb72fc0930522741d48 --- python/README.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/python/README.md b/python/README.md index 2a3e1ba9542f5..ed6c36bc085e9 100644 --- a/python/README.md +++ b/python/README.md @@ -48,7 +48,8 @@ python setup.py build_ext --inplace py.test pyarrow ``` -To change the build type, use the `--build-type` option: +To change the build type, use the `--build-type` option or set +`$PYARROW_BUILD_TYPE`: ```bash python setup.py build_ext --build-type=release --inplace @@ -57,6 +58,12 @@ python setup.py build_ext --build-type=release --inplace To pass through other build options to CMake, set the environment variable `$PYARROW_CMAKE_OPTIONS`. +#### Build the pyarrow Parquet file extension + +``` +export PYARROW_CMAKE_OPTIONS=-DPYARROW_BUILD_PARQUET=on +``` + #### Build the documentation ```bash From 3bd9a8d19ef4f681549a585407d3a4b298efd622 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Tue, 1 Nov 2016 17:21:46 -0400 Subject: [PATCH 3/4] Add --with-parquet option to setup.py Change-Id: Id0af78dc13d88aaeb4db33f8dfc9b8f932bddb2b --- python/README.md | 5 +++++ python/setup.py | 38 ++++++++++++++++++++++++-------------- 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/python/README.md b/python/README.md index ed6c36bc085e9..9ba0484445fb0 100644 --- a/python/README.md +++ b/python/README.md @@ -60,6 +60,9 @@ To pass through other build options to CMake, set the environment variable #### Build the pyarrow Parquet file extension +To build the integration with [parquet-cpp][1], pass `--with-parquet` to +setup.py or add `-DPYARROW_BUILD_PARQUET=on` to the general CMake options. + ``` export PYARROW_CMAKE_OPTIONS=-DPYARROW_BUILD_PARQUET=on ``` @@ -70,3 +73,5 @@ export PYARROW_CMAKE_OPTIONS=-DPYARROW_BUILD_PARQUET=on pip install -r doc/requirements.txt python setup.py build_sphinx ``` + +[1]: https://github.com/apache/parquet-cpp \ No newline at end of file diff --git a/python/setup.py b/python/setup.py index b3012e694243a..341cc64aa2cc8 100644 --- a/python/setup.py +++ b/python/setup.py @@ -97,13 +97,15 @@ def run(self): description = "Build the C-extensions for arrow" user_options = ([('extra-cmake-args=', None, 'extra arguments for CMake'), - ('build-type=', None, 'build type (debug or release)')] - + _build_ext.user_options) + ('build-type=', None, 'build type (debug or release)'), + ('with-parquet', None, 'build the Parquet extension')] + + _build_ext.user_options) def initialize_options(self): _build_ext.initialize_options(self) self.extra_cmake_args = os.environ.get('PYARROW_CMAKE_OPTIONS', '') self.build_type = os.environ.get('PYARROW_BUILD_TYPE', 'debug').lower() + self.with_parquet = False CYTHON_MODULE_NAMES = [ 'array', @@ -116,8 +118,6 @@ def initialize_options(self): 'schema', 'table'] - CYTHON_ALLOWED_FAILURES = ['parquet'] - def _run_cmake(self): # The directory containing this setup.py source = osp.dirname(osp.abspath(__file__)) @@ -141,17 +141,24 @@ def _run_cmake(self): if (cachedir != build_temp): return - pyexe_option = '-DPYTHON_EXECUTABLE=%s' % sys.executable static_lib_option = '' build_tests_option = '' - build_type_option = '-DCMAKE_BUILD_TYPE={0}'.format(self.build_type) + cmake_options = [ + '-DPYTHON_EXECUTABLE=%s' % sys.executable, + static_lib_option, + build_tests_option, + ] + + if self.with_parquet: + cmake_options.append('-DPYARROW_BUILD_PARQUET=on') if sys.platform != 'win32': - cmake_command = ['cmake', self.extra_cmake_args, pyexe_option, - build_tests_option, - build_type_option, - static_lib_option, source] + cmake_options.append('-DCMAKE_BUILD_TYPE={0}' + .format(self.build_type)) + + cmake_command = (['cmake', self.extra_cmake_args] + + cmake_options + [source]) self.spawn(cmake_command) args = ['make', 'VERBOSE=1'] @@ -166,10 +173,8 @@ def _run_cmake(self): # Generate the build files extra_cmake_args = shlex.split(self.extra_cmake_args) cmake_command = (['cmake'] + extra_cmake_args + + cmake_options + [source, - pyexe_option, - static_lib_option, - build_tests_option, '-G', cmake_generator]) if "-G" in self.extra_cmake_args: cmake_command = cmake_command[:-2] @@ -202,7 +207,7 @@ def _run_cmake(self): built_path = self.get_ext_built(name) if not os.path.exists(built_path): print(built_path) - if name in self.CYTHON_ALLOWED_FAILURES: + if self._failure_permitted(name): print('Cython module {0} failure permitted'.format(name)) continue raise RuntimeError('libpyarrow C-extension failed to build:', @@ -219,6 +224,11 @@ def _run_cmake(self): os.chdir(saved_cwd) + def _failure_permitted(self, name): + if name == 'parquet' and not self.with_parquet: + return True + return False + def _get_inplace_dir(self): pass From 07c05ccfd71cc14a926434f5ecf42535bab83f3a Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 2 Nov 2016 12:24:17 -0400 Subject: [PATCH 4/4] Update readme to illustrate proper use of with build_ext Change-Id: I3cd25f900a50ac7bc4d8ff56b15823dbf22b97c9 --- python/README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/python/README.md b/python/README.md index 9ba0484445fb0..4fce0d26b2850 100644 --- a/python/README.md +++ b/python/README.md @@ -61,7 +61,13 @@ To pass through other build options to CMake, set the environment variable #### Build the pyarrow Parquet file extension To build the integration with [parquet-cpp][1], pass `--with-parquet` to -setup.py or add `-DPYARROW_BUILD_PARQUET=on` to the general CMake options. +the `build_ext` option in setup.py: + +``` +python setup.py build_ext --with-parquet install +``` + +Alternately, add `-DPYARROW_BUILD_PARQUET=on` to the general CMake options. ``` export PYARROW_CMAKE_OPTIONS=-DPYARROW_BUILD_PARQUET=on