From 6fdcd4943ff9a8cc66afbee380217cec40c0cda0 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 9 Mar 2016 15:45:05 -0800 Subject: [PATCH] ARROW-54: [Python] Rename package to "pyarrow" Also fixed rpath issues (at great cost) per ARROW-53 Author: Wes McKinney Closes #23 from wesm/ARROW-54 and squashes the following commits: b8ce0e8 [Wes McKinney] Update installation instructions cae9b39 [Wes McKinney] Fix rpath issues per ARROW-53 7554539 [Wes McKinney] Twiddle rpath stuff, remove empty arrow_test_util module 8cca41a [Wes McKinney] Fix Travis CI script for renamed package 1d37c93 [Wes McKinney] Opt in to building unit tests 60088d0 [Wes McKinney] Rename package to pyarrow e3d0caf [Wes McKinney] Note on other Python interpreters 80d3bac [Wes McKinney] Start installation document --- .travis.yml | 4 +- ci/travis_script_python.sh | 2 +- cpp/CMakeLists.txt | 29 ++++--- cpp/src/arrow/CMakeLists.txt | 2 +- cpp/src/arrow/util/CMakeLists.txt | 44 ++++------ python/CMakeLists.txt | 31 ++++--- python/arrow/__init__.py | 38 -------- python/doc/INSTALL.md | 87 +++++++++++++++++++ python/pyarrow/__init__.py | 38 ++++++++ python/{arrow => pyarrow}/array.pxd | 8 +- python/{arrow => pyarrow}/array.pyx | 14 +-- python/{arrow => pyarrow}/compat.py | 0 python/{arrow => pyarrow}/config.pyx | 0 python/{arrow => pyarrow}/error.pxd | 2 +- python/{arrow => pyarrow}/error.pyx | 5 +- python/{arrow => pyarrow}/formatting.py | 2 +- .../{arrow => pyarrow}/includes/__init__.pxd | 0 python/{arrow => pyarrow}/includes/common.pxd | 0 .../includes/libarrow.pxd} | 2 +- .../{arrow => pyarrow}/includes/parquet.pxd | 2 +- .../{arrow => pyarrow}/includes/pyarrow.pxd | 6 +- python/{arrow => pyarrow}/parquet.pyx | 4 +- python/{arrow => pyarrow}/scalar.pxd | 6 +- python/{arrow => pyarrow}/scalar.pyx | 6 +- python/{arrow => pyarrow}/schema.pxd | 4 +- python/{arrow => pyarrow}/schema.pyx | 6 +- python/{arrow => pyarrow}/tests/__init__.py | 0 python/{arrow => pyarrow}/tests/test_array.py | 16 ++-- .../tests/test_convert_builtin.py | 52 +++++------ .../{arrow => pyarrow}/tests/test_scalars.py | 4 +- .../{arrow => pyarrow}/tests/test_schema.py | 4 +- python/requirements.txt | 1 - python/setup.py | 52 ++++++----- python/src/pyarrow/util/CMakeLists.txt | 30 ++++--- 34 files changed, 300 insertions(+), 201 deletions(-) delete mode 100644 python/arrow/__init__.py create mode 100644 python/doc/INSTALL.md create mode 100644 python/pyarrow/__init__.py rename python/{arrow => pyarrow}/array.pxd (90%) rename python/{arrow => pyarrow}/array.pyx (93%) rename python/{arrow => pyarrow}/compat.py (100%) rename python/{arrow => pyarrow}/config.pyx (100%) rename python/{arrow => pyarrow}/error.pxd (95%) rename python/{arrow => pyarrow}/error.pyx (92%) rename python/{arrow => pyarrow}/formatting.py (98%) rename python/{arrow => pyarrow}/includes/__init__.pxd (100%) rename python/{arrow => pyarrow}/includes/common.pxd (100%) rename python/{arrow/includes/arrow.pxd => pyarrow/includes/libarrow.pxd} (99%) rename python/{arrow => pyarrow}/includes/parquet.pxd (97%) rename python/{arrow => pyarrow}/includes/pyarrow.pxd (90%) rename python/{arrow => pyarrow}/parquet.pyx (91%) rename python/{arrow => pyarrow}/scalar.pxd (93%) rename python/{arrow => pyarrow}/scalar.pyx (97%) rename python/{arrow => pyarrow}/schema.pxd (91%) rename python/{arrow => pyarrow}/schema.pyx (97%) rename python/{arrow => pyarrow}/tests/__init__.py (100%) rename python/{arrow => pyarrow}/tests/test_array.py (80%) rename python/{arrow => pyarrow}/tests/test_convert_builtin.py (58%) rename python/{arrow => pyarrow}/tests/test_scalars.py (97%) rename python/{arrow => pyarrow}/tests/test_schema.py (96%) diff --git a/.travis.yml b/.travis.yml index 9e858d7d98e48..49a956ead3dca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,7 +27,8 @@ matrix: - $TRAVIS_BUILD_DIR/ci/travis_script_cpp.sh - $TRAVIS_BUILD_DIR/ci/travis_script_python.sh - compiler: clang - language: cpp + language: objective-c + osx_image: xcode6.4 os: osx addons: before_script: @@ -40,7 +41,6 @@ before_install: - ulimit -c unlimited -S - export CPP_BUILD_DIR=$TRAVIS_BUILD_DIR/cpp-build - export ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install -- export LD_LIBRARY_PATH=$ARROW_CPP_INSTALL/lib:$LD_LIBRARY_PATH after_script: - rm -rf $CPP_BUILD_DIR diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh index 9b0bd4f54cbc9..14d66b44ff8ee 100755 --- a/ci/travis_script_python.sh +++ b/ci/travis_script_python.sh @@ -48,7 +48,7 @@ export ARROW_HOME=$ARROW_CPP_INSTALL python setup.py build_ext --inplace -py.test -vv -r sxX arrow +py.test -vv -r sxX pyarrow # if [ $TRAVIS_OS_NAME == "linux" ]; then # valgrind --tool=memcheck py.test -vv -r sxX arrow diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index e8cb88c0b4d9b..f5f6038031127 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -37,11 +37,6 @@ if ("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1") set(CMAKE_EXPORT_COMPILE_COMMANDS 1) endif() -if(APPLE) - # In newer versions of CMake, this is the default setting - set(CMAKE_MACOSX_RPATH 1) -endif() - find_program(CCACHE_FOUND ccache) if(CCACHE_FOUND) set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache) @@ -339,10 +334,13 @@ endfunction() if ("$ENV{GTEST_HOME}" STREQUAL "") set(GTest_HOME ${THIRDPARTY_DIR}/googletest-release-1.7.0) endif() -find_package(GTest REQUIRED) -include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) -ADD_THIRDPARTY_LIB(gtest - STATIC_LIB ${GTEST_STATIC_LIB}) + +if(ARROW_BUILD_TESTS) + find_package(GTest REQUIRED) + include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) + ADD_THIRDPARTY_LIB(gtest + STATIC_LIB ${GTEST_STATIC_LIB}) +endif() ## Google PerfTools ## @@ -366,7 +364,7 @@ ADD_THIRDPARTY_LIB(gtest ############################################################ # Linker setup ############################################################ -set(ARROW_MIN_TEST_LIBS arrow arrow_test_main arrow_test_util ${ARROW_BASE_LIBS}) +set(ARROW_MIN_TEST_LIBS arrow arrow_test_main ${ARROW_BASE_LIBS}) set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS}) ############################################################ @@ -461,9 +459,18 @@ add_library(arrow ${LIBARROW_LINKAGE} ${ARROW_SRCS} ) + +if (APPLE) + set_target_properties(arrow + PROPERTIES + BUILD_WITH_INSTALL_RPATH ON + INSTALL_NAME_DIR "@rpath") +endif() + set_target_properties(arrow PROPERTIES - LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}") + LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}" +) target_link_libraries(arrow ${LIBARROW_LINK_LIBS}) add_subdirectory(src/arrow) diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt index 77326ce38d754..73e6a9b22c94a 100644 --- a/cpp/src/arrow/CMakeLists.txt +++ b/cpp/src/arrow/CMakeLists.txt @@ -27,6 +27,6 @@ install(FILES # Unit tests ####################################### -set(ARROW_TEST_LINK_LIBS arrow_test_util ${ARROW_MIN_TEST_LIBS}) +set(ARROW_TEST_LINK_LIBS ${ARROW_MIN_TEST_LIBS}) ADD_ARROW_TEST(array-test) diff --git a/cpp/src/arrow/util/CMakeLists.txt b/cpp/src/arrow/util/CMakeLists.txt index 4272ce4285482..d8e2f98f2c85e 100644 --- a/cpp/src/arrow/util/CMakeLists.txt +++ b/cpp/src/arrow/util/CMakeLists.txt @@ -28,37 +28,27 @@ install(FILES status.h DESTINATION include/arrow/util) -####################################### -# arrow_test_util -####################################### - -add_library(arrow_test_util) -target_link_libraries(arrow_test_util -) - -SET_TARGET_PROPERTIES(arrow_test_util PROPERTIES LINKER_LANGUAGE CXX) - ####################################### # arrow_test_main ####################################### -add_library(arrow_test_main - test_main.cc) - -if (APPLE) - target_link_libraries(arrow_test_main - gtest - arrow_test_util - dl) - set_target_properties(arrow_test_main - PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") -else() - target_link_libraries(arrow_test_main - gtest - arrow_test_util - pthread - dl - ) +if (ARROW_BUILD_TESTS) + add_library(arrow_test_main + test_main.cc) + + if (APPLE) + target_link_libraries(arrow_test_main + gtest + dl) + set_target_properties(arrow_test_main + PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") + else() + target_link_libraries(arrow_test_main + gtest + pthread + dl + ) + endif() endif() ADD_ARROW_TEST(bit-util-test) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index 8f5c27b0f76d7..0ecafc7202e89 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -45,6 +45,13 @@ if ("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1") set(CMAKE_EXPORT_COMPILE_COMMANDS 1) endif() +# Top level cmake dir +if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") + option(PYARROW_BUILD_TESTS + "Build the PyArrow C++ googletest unit tests" + OFF) +endif() + find_program(CCACHE_FOUND ccache) if(CCACHE_FOUND) set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache) @@ -322,10 +329,12 @@ function(ADD_THIRDPARTY_LIB LIB_NAME) endfunction() ## GMock -find_package(GTest REQUIRED) -include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) -ADD_THIRDPARTY_LIB(gtest - STATIC_LIB ${GTEST_STATIC_LIB}) +if (PYARROW_BUILD_TESTS) + find_package(GTest REQUIRED) + include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) + ADD_THIRDPARTY_LIB(gtest + STATIC_LIB ${GTEST_STATIC_LIB}) +endif() ## Arrow find_package(Arrow REQUIRED) @@ -391,6 +400,10 @@ endif (UNIX) # Subdirectories ############################################################ +if (UNIX) + set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) +endif() + add_subdirectory(src/pyarrow) add_subdirectory(src/pyarrow/util) @@ -407,10 +420,11 @@ set(LINK_LIBS arrow ) +SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) + add_library(pyarrow SHARED ${PYARROW_SRCS}) target_link_libraries(pyarrow ${LINK_LIBS}) -set_target_properties(pyarrow PROPERTIES LINKER_LANGUAGE CXX) if(APPLE) set_target_properties(pyarrow PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") @@ -420,9 +434,6 @@ endif() # Setup and build Cython modules ############################################################ -set(USE_RELATIVE_RPATH ON) -set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE) - set(CYTHON_EXTENSIONS array config @@ -437,7 +448,7 @@ foreach(module ${CYTHON_EXTENSIONS}) list(REMOVE_AT directories -1) string(REPLACE "." "/" module_root "${module}") - set(module_SRC arrow/${module_root}.pyx) + set(module_SRC pyarrow/${module_root}.pyx) set_source_files_properties(${module_SRC} PROPERTIES CYTHON_IS_CXX 1) cython_add_module(${module_name} @@ -463,7 +474,7 @@ foreach(module ${CYTHON_EXTENSIONS}) endwhile(${i} GREATER 0) # for inplace development for now - set(module_install_rpath "${CMAKE_SOURCE_DIR}/arrow/") + #set(module_install_rpath "${CMAKE_SOURCE_DIR}/pyarrow/") set_target_properties(${module_name} PROPERTIES INSTALL_RPATH ${module_install_rpath}) diff --git a/python/arrow/__init__.py b/python/arrow/__init__.py deleted file mode 100644 index 3507ea0235afe..0000000000000 --- a/python/arrow/__init__.py +++ /dev/null @@ -1,38 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# flake8: noqa - -from arrow.array import (Array, from_pylist, total_allocated_bytes, - BooleanArray, NumericArray, - Int8Array, UInt8Array, - ListArray, StringArray) - -from arrow.error import ArrowException - -from arrow.scalar import (ArrayValue, Scalar, NA, NAType, - BooleanValue, - Int8Value, Int16Value, Int32Value, Int64Value, - UInt8Value, UInt16Value, UInt32Value, UInt64Value, - FloatValue, DoubleValue, ListValue, StringValue) - -from arrow.schema import (null, bool_, - int8, int16, int32, int64, - uint8, uint16, uint32, uint64, - float_, double, string, - list_, struct, field, - DataType, Field, Schema) diff --git a/python/doc/INSTALL.md b/python/doc/INSTALL.md new file mode 100644 index 0000000000000..d30a03046eda7 --- /dev/null +++ b/python/doc/INSTALL.md @@ -0,0 +1,87 @@ +## Building pyarrow (Apache Arrow Python library) + +First, clone the master git repository: + +```bash +git clone https://github.com/apache/arrow.git arrow +``` + +#### System requirements + +Building pyarrow requires: + +* A C++11 compiler + + * Linux: gcc >= 4.8 or clang >= 3.5 + * OS X: XCode 6.4 or higher preferred + +* [cmake][1] + +#### Python requirements + +You will need Python (CPython) 2.7, 3.4, or 3.5 installed. Earlier releases and +are not being targeted. + +> This library targets CPython only due to an emphasis on interoperability with +> pandas and NumPy, which are only available for CPython. + +The build requires NumPy, Cython, and a few other Python dependencies: + +```bash +pip install cython +cd arrow/python +pip install -r requirements.txt +``` + +#### Installing Arrow C++ library + +First, you should choose an installation location for Arrow C++. In the future +using the default system install location will work, but for now we are being +explicit: + +```bash +export ARROW_HOME=$HOME/local +``` + +Now, we build Arrow: + +```bash +cd arrow/cpp + +mkdir dev-build +cd dev-build + +cmake -DCMAKE_INSTALL_PREFIX=$ARROW_HOME .. + +make + +# Use sudo here if $ARROW_HOME requires it +make install +``` + +#### Install `pyarrow` + +```bash +cd arrow/python + +python setup.py install +``` + +> On XCode 6 and prior there are some known OS X `@rpath` issues. If you are +> unable to import pyarrow, upgrading XCode may be the solution. + + +```python +In [1]: import pyarrow + +In [2]: pyarrow.from_pylist([1,2,3]) +Out[2]: + +[ + 1, + 2, + 3 +] +``` + +[1]: https://cmake.org/ \ No newline at end of file diff --git a/python/pyarrow/__init__.py b/python/pyarrow/__init__.py new file mode 100644 index 0000000000000..8d93a156bcc3d --- /dev/null +++ b/python/pyarrow/__init__.py @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# flake8: noqa + +from pyarrow.array import (Array, from_pylist, total_allocated_bytes, + BooleanArray, NumericArray, + Int8Array, UInt8Array, + ListArray, StringArray) + +from pyarrow.error import ArrowException + +from pyarrow.scalar import (ArrayValue, Scalar, NA, NAType, + BooleanValue, + Int8Value, Int16Value, Int32Value, Int64Value, + UInt8Value, UInt16Value, UInt32Value, UInt64Value, + FloatValue, DoubleValue, ListValue, StringValue) + +from pyarrow.schema import (null, bool_, + int8, int16, int32, int64, + uint8, uint16, uint32, uint64, + float_, double, string, + list_, struct, field, + DataType, Field, Schema) diff --git a/python/arrow/array.pxd b/python/pyarrow/array.pxd similarity index 90% rename from python/arrow/array.pxd rename to python/pyarrow/array.pxd index 482f8f796dd26..d0d3486c032fe 100644 --- a/python/arrow/array.pxd +++ b/python/pyarrow/array.pxd @@ -15,12 +15,12 @@ # specific language governing permissions and limitations # under the License. -from arrow.includes.common cimport shared_ptr -from arrow.includes.arrow cimport CArray, LogicalType +from pyarrow.includes.common cimport shared_ptr +from pyarrow.includes.libarrow cimport CArray, LogicalType -from arrow.scalar import NA +from pyarrow.scalar import NA -from arrow.schema cimport DataType +from pyarrow.schema cimport DataType cdef extern from "Python.h": int PySlice_Check(object) diff --git a/python/arrow/array.pyx b/python/pyarrow/array.pyx similarity index 93% rename from python/arrow/array.pyx rename to python/pyarrow/array.pyx index b367e3b84a8b3..bceb333c94ea5 100644 --- a/python/arrow/array.pyx +++ b/python/pyarrow/array.pyx @@ -19,14 +19,14 @@ # distutils: language = c++ # cython: embedsignature = True -from arrow.includes.arrow cimport * -cimport arrow.includes.pyarrow as pyarrow +from pyarrow.includes.libarrow cimport * +cimport pyarrow.includes.pyarrow as pyarrow -from arrow.compat import frombytes, tobytes -from arrow.error cimport check_status +from pyarrow.compat import frombytes, tobytes +from pyarrow.error cimport check_status -cimport arrow.scalar as scalar -from arrow.scalar import NA +cimport pyarrow.scalar as scalar +from pyarrow.scalar import NA def total_allocated_bytes(): cdef MemoryPool* pool = pyarrow.GetMemoryPool() @@ -52,7 +52,7 @@ cdef class Array: raise StopIteration def __repr__(self): - from arrow.formatting import array_format + from pyarrow.formatting import array_format type_format = object.__repr__(self) values = array_format(self, window=10) return '{0}\n{1}'.format(type_format, values) diff --git a/python/arrow/compat.py b/python/pyarrow/compat.py similarity index 100% rename from python/arrow/compat.py rename to python/pyarrow/compat.py diff --git a/python/arrow/config.pyx b/python/pyarrow/config.pyx similarity index 100% rename from python/arrow/config.pyx rename to python/pyarrow/config.pyx diff --git a/python/arrow/error.pxd b/python/pyarrow/error.pxd similarity index 95% rename from python/arrow/error.pxd rename to python/pyarrow/error.pxd index c18cb3efffca6..d226abeda04e0 100644 --- a/python/arrow/error.pxd +++ b/python/pyarrow/error.pxd @@ -15,6 +15,6 @@ # specific language governing permissions and limitations # under the License. -from arrow.includes.pyarrow cimport * +from pyarrow.includes.pyarrow cimport * cdef check_status(const Status& status) diff --git a/python/arrow/error.pyx b/python/pyarrow/error.pyx similarity index 92% rename from python/arrow/error.pyx rename to python/pyarrow/error.pyx index f1d516358819d..3f8d7dd646091 100644 --- a/python/arrow/error.pyx +++ b/python/pyarrow/error.pyx @@ -15,9 +15,8 @@ # specific language governing permissions and limitations # under the License. -from arrow.includes.common cimport c_string - -from arrow.compat import frombytes +from pyarrow.includes.common cimport c_string +from pyarrow.compat import frombytes class ArrowException(Exception): pass diff --git a/python/arrow/formatting.py b/python/pyarrow/formatting.py similarity index 98% rename from python/arrow/formatting.py rename to python/pyarrow/formatting.py index a42d4e4bb5713..5fe0611f8450b 100644 --- a/python/arrow/formatting.py +++ b/python/pyarrow/formatting.py @@ -17,7 +17,7 @@ # Pretty-printing and other formatting utilities for Arrow data structures -import arrow.scalar as scalar +import pyarrow.scalar as scalar def array_format(arr, window=None): diff --git a/python/arrow/includes/__init__.pxd b/python/pyarrow/includes/__init__.pxd similarity index 100% rename from python/arrow/includes/__init__.pxd rename to python/pyarrow/includes/__init__.pxd diff --git a/python/arrow/includes/common.pxd b/python/pyarrow/includes/common.pxd similarity index 100% rename from python/arrow/includes/common.pxd rename to python/pyarrow/includes/common.pxd diff --git a/python/arrow/includes/arrow.pxd b/python/pyarrow/includes/libarrow.pxd similarity index 99% rename from python/arrow/includes/arrow.pxd rename to python/pyarrow/includes/libarrow.pxd index 0cc44c06cb607..baba112833e0d 100644 --- a/python/arrow/includes/arrow.pxd +++ b/python/pyarrow/includes/libarrow.pxd @@ -17,7 +17,7 @@ # distutils: language = c++ -from arrow.includes.common cimport * +from pyarrow.includes.common cimport * cdef extern from "arrow/api.h" namespace "arrow" nogil: diff --git a/python/arrow/includes/parquet.pxd b/python/pyarrow/includes/parquet.pxd similarity index 97% rename from python/arrow/includes/parquet.pxd rename to python/pyarrow/includes/parquet.pxd index 62342f3066969..99a2d423d9cba 100644 --- a/python/arrow/includes/parquet.pxd +++ b/python/pyarrow/includes/parquet.pxd @@ -17,7 +17,7 @@ # distutils: language = c++ -from arrow.includes.common cimport * +from pyarrow.includes.common cimport * cdef extern from "parquet/api/reader.h" namespace "parquet_cpp" nogil: cdef cppclass ColumnReader: diff --git a/python/arrow/includes/pyarrow.pxd b/python/pyarrow/includes/pyarrow.pxd similarity index 90% rename from python/arrow/includes/pyarrow.pxd rename to python/pyarrow/includes/pyarrow.pxd index 3eed5b8542493..9a0c004b7684a 100644 --- a/python/arrow/includes/pyarrow.pxd +++ b/python/pyarrow/includes/pyarrow.pxd @@ -17,9 +17,9 @@ # distutils: language = c++ -from arrow.includes.common cimport * -from arrow.includes.arrow cimport (CArray, CDataType, LogicalType, - MemoryPool) +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport (CArray, CDataType, LogicalType, + MemoryPool) cdef extern from "pyarrow/api.h" namespace "pyarrow" nogil: # We can later add more of the common status factory methods as needed diff --git a/python/arrow/parquet.pyx b/python/pyarrow/parquet.pyx similarity index 91% rename from python/arrow/parquet.pyx rename to python/pyarrow/parquet.pyx index 23c3838bcad1f..622e7d0772456 100644 --- a/python/arrow/parquet.pyx +++ b/python/pyarrow/parquet.pyx @@ -19,5 +19,5 @@ # distutils: language = c++ # cython: embedsignature = True -from arrow.compat import frombytes, tobytes -from arrow.includes.parquet cimport * +from pyarrow.compat import frombytes, tobytes +from pyarrow.includes.parquet cimport * diff --git a/python/arrow/scalar.pxd b/python/pyarrow/scalar.pxd similarity index 93% rename from python/arrow/scalar.pxd rename to python/pyarrow/scalar.pxd index 4e0a3647155a6..b06845718649b 100644 --- a/python/arrow/scalar.pxd +++ b/python/pyarrow/scalar.pxd @@ -15,10 +15,10 @@ # specific language governing permissions and limitations # under the License. -from arrow.includes.common cimport * -from arrow.includes.arrow cimport * +from pyarrow.includes.common cimport * +from pyarrow.includes.libarrow cimport * -from arrow.schema cimport DataType +from pyarrow.schema cimport DataType cdef class Scalar: cdef readonly: diff --git a/python/arrow/scalar.pyx b/python/pyarrow/scalar.pyx similarity index 97% rename from python/arrow/scalar.pyx rename to python/pyarrow/scalar.pyx index 72a280e334f4e..261a38967c495 100644 --- a/python/arrow/scalar.pyx +++ b/python/pyarrow/scalar.pyx @@ -15,10 +15,10 @@ # specific language governing permissions and limitations # under the License. -from arrow.schema cimport DataType, box_data_type +from pyarrow.schema cimport DataType, box_data_type -from arrow.compat import frombytes -import arrow.schema as schema +from pyarrow.compat import frombytes +import pyarrow.schema as schema NA = None diff --git a/python/arrow/schema.pxd b/python/pyarrow/schema.pxd similarity index 91% rename from python/arrow/schema.pxd rename to python/pyarrow/schema.pxd index 8cc244aaba341..07b9bd04da20e 100644 --- a/python/arrow/schema.pxd +++ b/python/pyarrow/schema.pxd @@ -15,8 +15,8 @@ # specific language governing permissions and limitations # under the License. -from arrow.includes.common cimport shared_ptr -from arrow.includes.arrow cimport CDataType, CField, CSchema +from pyarrow.includes.common cimport shared_ptr +from pyarrow.includes.libarrow cimport CDataType, CField, CSchema cdef class DataType: cdef: diff --git a/python/arrow/schema.pyx b/python/pyarrow/schema.pyx similarity index 97% rename from python/arrow/schema.pyx rename to python/pyarrow/schema.pyx index 3001531eb607d..ea878720d5bb8 100644 --- a/python/arrow/schema.pyx +++ b/python/pyarrow/schema.pyx @@ -22,9 +22,9 @@ # distutils: language = c++ # cython: embedsignature = True -from arrow.compat import frombytes, tobytes -from arrow.includes.arrow cimport * -cimport arrow.includes.pyarrow as pyarrow +from pyarrow.compat import frombytes, tobytes +from pyarrow.includes.libarrow cimport * +cimport pyarrow.includes.pyarrow as pyarrow cimport cpython diff --git a/python/arrow/tests/__init__.py b/python/pyarrow/tests/__init__.py similarity index 100% rename from python/arrow/tests/__init__.py rename to python/pyarrow/tests/__init__.py diff --git a/python/arrow/tests/test_array.py b/python/pyarrow/tests/test_array.py similarity index 80% rename from python/arrow/tests/test_array.py rename to python/pyarrow/tests/test_array.py index ebd872c744e44..034c1576551d3 100644 --- a/python/arrow/tests/test_array.py +++ b/python/pyarrow/tests/test_array.py @@ -15,19 +15,19 @@ # specific language governing permissions and limitations # under the License. -from arrow.compat import unittest -import arrow -import arrow.formatting as fmt +from pyarrow.compat import unittest +import pyarrow +import pyarrow.formatting as fmt class TestArrayAPI(unittest.TestCase): def test_getitem_NA(self): - arr = arrow.from_pylist([1, None, 2]) - assert arr[1] is arrow.NA + arr = pyarrow.from_pylist([1, None, 2]) + assert arr[1] is pyarrow.NA def test_list_format(self): - arr = arrow.from_pylist([[1], None, [2, 3]]) + arr = pyarrow.from_pylist([[1], None, [2, 3]]) result = fmt.array_format(arr) expected = """\ [ @@ -39,7 +39,7 @@ def test_list_format(self): assert result == expected def test_string_format(self): - arr = arrow.from_pylist(['foo', None, 'bar']) + arr = pyarrow.from_pylist(['foo', None, 'bar']) result = fmt.array_format(arr) expected = """\ [ @@ -50,7 +50,7 @@ def test_string_format(self): assert result == expected def test_long_array_format(self): - arr = arrow.from_pylist(range(100)) + arr = pyarrow.from_pylist(range(100)) result = fmt.array_format(arr, window=2) expected = """\ [ diff --git a/python/arrow/tests/test_convert_builtin.py b/python/pyarrow/tests/test_convert_builtin.py similarity index 58% rename from python/arrow/tests/test_convert_builtin.py rename to python/pyarrow/tests/test_convert_builtin.py index 57e6ab9f0e7b5..25f696912105d 100644 --- a/python/arrow/tests/test_convert_builtin.py +++ b/python/pyarrow/tests/test_convert_builtin.py @@ -15,8 +15,8 @@ # specific language governing permissions and limitations # under the License. -from arrow.compat import unittest -import arrow +from pyarrow.compat import unittest +import pyarrow class TestConvertList(unittest.TestCase): @@ -25,61 +25,61 @@ def test_boolean(self): pass def test_empty_list(self): - arr = arrow.from_pylist([]) + arr = pyarrow.from_pylist([]) assert len(arr) == 0 assert arr.null_count == 0 - assert arr.type == arrow.null() + assert arr.type == pyarrow.null() def test_all_none(self): - arr = arrow.from_pylist([None, None]) + arr = pyarrow.from_pylist([None, None]) assert len(arr) == 2 assert arr.null_count == 2 - assert arr.type == arrow.null() + assert arr.type == pyarrow.null() def test_integer(self): - arr = arrow.from_pylist([1, None, 3, None]) + arr = pyarrow.from_pylist([1, None, 3, None]) assert len(arr) == 4 assert arr.null_count == 2 - assert arr.type == arrow.int64() + assert arr.type == pyarrow.int64() def test_garbage_collection(self): import gc - bytes_before = arrow.total_allocated_bytes() - arrow.from_pylist([1, None, 3, None]) + bytes_before = pyarrow.total_allocated_bytes() + pyarrow.from_pylist([1, None, 3, None]) gc.collect() - assert arrow.total_allocated_bytes() == bytes_before + assert pyarrow.total_allocated_bytes() == bytes_before def test_double(self): data = [1.5, 1, None, 2.5, None, None] - arr = arrow.from_pylist(data) + arr = pyarrow.from_pylist(data) assert len(arr) == 6 assert arr.null_count == 3 - assert arr.type == arrow.double() + assert arr.type == pyarrow.double() def test_string(self): data = ['foo', b'bar', None, 'arrow'] - arr = arrow.from_pylist(data) + arr = pyarrow.from_pylist(data) assert len(arr) == 4 assert arr.null_count == 1 - assert arr.type == arrow.string() + assert arr.type == pyarrow.string() def test_mixed_nesting_levels(self): - arrow.from_pylist([1, 2, None]) - arrow.from_pylist([[1], [2], None]) - arrow.from_pylist([[1], [2], [None]]) + pyarrow.from_pylist([1, 2, None]) + pyarrow.from_pylist([[1], [2], None]) + pyarrow.from_pylist([[1], [2], [None]]) - with self.assertRaises(arrow.ArrowException): - arrow.from_pylist([1, 2, [1]]) + with self.assertRaises(pyarrow.ArrowException): + pyarrow.from_pylist([1, 2, [1]]) - with self.assertRaises(arrow.ArrowException): - arrow.from_pylist([1, 2, []]) + with self.assertRaises(pyarrow.ArrowException): + pyarrow.from_pylist([1, 2, []]) - with self.assertRaises(arrow.ArrowException): - arrow.from_pylist([[1], [2], [None, [1]]]) + with self.assertRaises(pyarrow.ArrowException): + pyarrow.from_pylist([[1], [2], [None, [1]]]) def test_list_of_int(self): data = [[1, 2, 3], [], None, [1, 2]] - arr = arrow.from_pylist(data) + arr = pyarrow.from_pylist(data) assert len(arr) == 4 assert arr.null_count == 1 - assert arr.type == arrow.list_(arrow.int64()) + assert arr.type == pyarrow.list_(pyarrow.int64()) diff --git a/python/arrow/tests/test_scalars.py b/python/pyarrow/tests/test_scalars.py similarity index 97% rename from python/arrow/tests/test_scalars.py rename to python/pyarrow/tests/test_scalars.py index 951380bd981e4..021737db6726e 100644 --- a/python/arrow/tests/test_scalars.py +++ b/python/pyarrow/tests/test_scalars.py @@ -15,8 +15,8 @@ # specific language governing permissions and limitations # under the License. -from arrow.compat import unittest, u -import arrow +from pyarrow.compat import unittest, u +import pyarrow as arrow class TestScalars(unittest.TestCase): diff --git a/python/arrow/tests/test_schema.py b/python/pyarrow/tests/test_schema.py similarity index 96% rename from python/arrow/tests/test_schema.py rename to python/pyarrow/tests/test_schema.py index a89edd74a0adf..0235526198f35 100644 --- a/python/arrow/tests/test_schema.py +++ b/python/pyarrow/tests/test_schema.py @@ -15,8 +15,8 @@ # specific language governing permissions and limitations # under the License. -from arrow.compat import unittest -import arrow +from pyarrow.compat import unittest +import pyarrow as arrow class TestTypes(unittest.TestCase): diff --git a/python/requirements.txt b/python/requirements.txt index a82cb20aab86e..f42c90c5c9b3f 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,4 +1,3 @@ pytest numpy>=1.7.0 -pandas>=0.12.0 six diff --git a/python/setup.py b/python/setup.py index eb3ff2a1547d6..5cc871aba9f81 100644 --- a/python/setup.py +++ b/python/setup.py @@ -27,7 +27,7 @@ import sys import pkg_resources -from setuptools import setup +from setuptools import setup, Extension import os @@ -40,10 +40,12 @@ is_64_bit = sys.maxsize > 2**32 # Check if this is a debug build of Python. -if hasattr(sys, 'gettotalrefcount'): - build_type = 'Debug' -else: - build_type = 'Release' +# if hasattr(sys, 'gettotalrefcount'): +# build_type = 'Debug' +# else: +# build_type = 'Release' + +build_type = 'Debug' if Cython.__version__ < '0.19.1': raise Exception('Please upgrade to Cython 0.19.1 or newer') @@ -51,7 +53,7 @@ MAJOR = 0 MINOR = 1 MICRO = 0 -VERSION = '%d.%d.%d' % (MAJOR, MINOR, MICRO) +VERSION = '%d.%d.%ddev' % (MAJOR, MINOR, MICRO) class clean(_clean): @@ -70,6 +72,9 @@ class build_ext(_build_ext): def build_extensions(self): numpy_incl = pkg_resources.resource_filename('numpy', 'core/include') + self.extensions = [ext for ext in self.extensions + if ext.name != '__dummy__'] + for ext in self.extensions: if (hasattr(ext, 'include_dirs') and numpy_incl not in ext.include_dirs): @@ -98,6 +103,7 @@ def _run_cmake(self): # The staging directory for the module being built build_temp = pjoin(os.getcwd(), self.build_temp) + build_lib = os.path.join(os.getcwd(), self.build_lib) # Change to the build directory saved_cwd = os.getcwd() @@ -124,7 +130,7 @@ def _run_cmake(self): static_lib_option, source] self.spawn(cmake_command) - args = ['make'] + args = ['make', 'VERBOSE=1'] if 'PYARROW_PARALLEL' in os.environ: args.append('-j{0}'.format(os.environ['PYARROW_PARALLEL'])) self.spawn(args) @@ -150,21 +156,19 @@ def _run_cmake(self): if self.inplace: # a bit hacky build_lib = saved_cwd - else: - build_lib = pjoin(os.getcwd(), self.build_lib) # Move the built libpyarrow library to the place expected by the Python # build if sys.platform != 'win32': name, = glob.glob('libpyarrow.*') try: - os.makedirs(pjoin(build_lib, 'arrow')) + os.makedirs(pjoin(build_lib, 'pyarrow')) except OSError: pass - shutil.move(name, pjoin(build_lib, 'arrow', name)) + shutil.move(name, pjoin(build_lib, 'pyarrow', name)) else: shutil.move(pjoin(build_type, 'pyarrow.dll'), - pjoin(build_lib, 'arrow', 'pyarrow.dll')) + pjoin(build_lib, 'pyarrow', 'pyarrow.dll')) # Move the built C-extension to the place expected by the Python build self._found_names = [] @@ -192,7 +196,7 @@ def _get_inplace_dir(self): def _get_cmake_ext_path(self, name): # Get the package directory from build_py build_py = self.get_finalized_command('build_py') - package_dir = build_py.get_package_dir('arrow') + package_dir = build_py.get_package_dir('pyarrow') # This is the name of the arrow C-extension suffix = sysconfig.get_config_var('EXT_SUFFIX') if suffix is None: @@ -217,23 +221,23 @@ def get_names(self): def get_outputs(self): # Just the C extensions - cmake_exts = [self._get_cmake_ext_path(name) - for name in self.get_names()] - regular_exts = _build_ext.get_outputs(self) - return regular_exts + cmake_exts + # regular_exts = _build_ext.get_outputs(self) + return [self._get_cmake_ext_path(name) + for name in self.get_names()] -extensions = [] - DESC = """\ Python library for Apache Arrow""" setup( - name="arrow", - packages=['arrow', 'arrow.tests'], + name="pyarrow", + packages=['pyarrow', 'pyarrow.tests'], version=VERSION, - package_data={'arrow': ['*.pxd', '*.pyx']}, - ext_modules=extensions, + zip_safe=False, + package_data={'pyarrow': ['*.pxd', '*.pyx']}, + # Dummy extension to trigger build_ext + ext_modules=[Extension('__dummy__', sources=[])], + cmdclass={ 'clean': clean, 'build_ext': build_ext @@ -243,5 +247,5 @@ def get_outputs(self): license='Apache License, Version 2.0', maintainer="Apache Arrow Developers", maintainer_email="dev@arrow.apache.org", - test_suite="arrow.tests" + test_suite="pyarrow.tests" ) diff --git a/python/src/pyarrow/util/CMakeLists.txt b/python/src/pyarrow/util/CMakeLists.txt index 3fd8bac31506d..4afb4d0f912b1 100644 --- a/python/src/pyarrow/util/CMakeLists.txt +++ b/python/src/pyarrow/util/CMakeLists.txt @@ -19,19 +19,21 @@ # pyarrow_test_main ####################################### -add_library(pyarrow_test_main - test_main.cc) +if (PYARROW_BUILD_TESTS) + add_library(pyarrow_test_main + test_main.cc) -if (APPLE) - target_link_libraries(pyarrow_test_main - gtest - dl) - set_target_properties(pyarrow_test_main - PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") -else() - target_link_libraries(pyarrow_test_main - gtest - pthread - dl - ) + if (APPLE) + target_link_libraries(pyarrow_test_main + gtest + dl) + set_target_properties(pyarrow_test_main + PROPERTIES LINK_FLAGS "-undefined dynamic_lookup") + else() + target_link_libraries(pyarrow_test_main + gtest + pthread + dl + ) + endif() endif()