diff --git a/.travis.yml b/.travis.yml index d919baff6d3..5bd27abb799 100644 --- a/.travis.yml +++ b/.travis.yml @@ -261,8 +261,20 @@ matrix: after_failure: -- COREFILE=$(find . -maxdepth 2 -name "core*" | head -n 1) -- if [[ -f "$COREFILE" ]]; then gdb -c "$COREFILE" example -ex "thread apply all bt" -ex "set pagination 0" -batch; fi +- | + if [ "$TRAVIS_OS_NAME" = "osx" ]; then + COREFILE=$(find /cores -maxdepth 1 -type f -name "core.*" | head -n 1) + if [[ -f "$COREFILE" ]]; then + lldb -c "$COREFILE" --batch --one-line "thread backtrace all -e true" + fi + ls -la ~/Library/Logs/DiagnosticReports/ + cat ~/Library/Logs/DiagnosticReports/*.crash + else + COREFILE=$(find . -maxdepth 2 -name "core*" | head -n 1) + if [[ -f "$COREFILE" ]]; then + gdb -c "$COREFILE" example -ex "thread apply all bt" -ex "set pagination 0" -batch + fi + fi env: global: diff --git a/ci/test-cmake-build-script.bat b/ci/test-cmake-build-script.bat index 94857e82333..c2740b89650 100644 --- a/ci/test-cmake-build-script.bat +++ b/ci/test-cmake-build-script.bat @@ -39,6 +39,11 @@ cmake -G "%GENERATOR%" ^ FINDSTR /M /C:"Could not find the Flatbuffers library" error.txt || exit /B set FLATBUFFERS_HOME= +popd +rmdir /S /Q %BUILD_DIR% +mkdir %BUILD_DIR% +pushd %BUILD_DIR% + echo Test cmake script errors out on gflags missed set GFLAGS_HOME=WrongPath @@ -51,6 +56,11 @@ cmake -G "%GENERATOR%" ^ FINDSTR /M /C:"No static or shared library provided for gflags" error.txt || exit /B set GFLAGS_HOME= +popd +rmdir /S /Q %BUILD_DIR% +mkdir %BUILD_DIR% +pushd %BUILD_DIR% + echo Test cmake script errors out on snappy missed set SNAPPY_HOME=WrongPath @@ -63,6 +73,11 @@ cmake -G "%GENERATOR%" ^ FINDSTR /M /C:"Could not find the Snappy library" error.txt || exit /B set SNAPPY_HOME= +popd +rmdir /S /Q %BUILD_DIR% +mkdir %BUILD_DIR% +pushd %BUILD_DIR% + echo Test cmake script errors out on zlib missed set ZLIB_HOME=WrongPath @@ -75,6 +90,11 @@ cmake -G "%GENERATOR%" ^ FINDSTR /M /C:"Could not find the ZLIB library" error.txt || exit /B set ZLIB_HOME= +popd +rmdir /S /Q %BUILD_DIR% +mkdir %BUILD_DIR% +pushd %BUILD_DIR% + echo Test cmake script errors out on brotli missed set BROTLI_HOME=WrongPath @@ -87,6 +107,11 @@ cmake -G "%GENERATOR%" ^ FINDSTR /M /C:"Could not find the Brotli library" error.txt || exit /B set BROTLI_HOME= +popd +rmdir /S /Q %BUILD_DIR% +mkdir %BUILD_DIR% +pushd %BUILD_DIR% + echo Test cmake script errors out on lz4 missed set LZ4_HOME=WrongPath @@ -99,6 +124,11 @@ cmake -G "%GENERATOR%" ^ FINDSTR /M /C:"No static or shared library provided for lz4_static" error.txt || exit /B set LZ4_HOME= +popd +rmdir /S /Q %BUILD_DIR% +mkdir %BUILD_DIR% +pushd %BUILD_DIR% + echo Test cmake script errors out on zstd missed set ZSTD_HOME=WrongPath @@ -134,10 +164,15 @@ cmake -G "%GENERATOR%" ^ .. 2>output.txt set LIBRARY_FOUND_MSG=Added static library dependency -for %%x in (snappy gflags zlib brotli_enc brotli_dec brotli_common lz4_static zstd_static) do ( +for %%x in (snappy gflags brotli_enc brotli_dec brotli_common lz4_static zstd_static) do ( + echo Checking %%x library path + FINDSTR /C:"%LIBRARY_FOUND_MSG% %%x: %CONDA_PREFIX:\=/%" output.txt || exit /B +) +set LIBRARY_FOUND_MSG=Added shared library dependency +for %%x in (zlib) do ( echo Checking %%x library path FINDSTR /C:"%LIBRARY_FOUND_MSG% %%x: %CONDA_PREFIX:\=/%" output.txt || exit /B ) popd -rmdir /S /Q %BUILD_DIR% \ No newline at end of file +rmdir /S /Q %BUILD_DIR% diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh index e12a37391c7..8238dbd00af 100755 --- a/ci/travis_script_python.sh +++ b/ci/travis_script_python.sh @@ -35,6 +35,15 @@ CONDA_ENV_DIR=$TRAVIS_BUILD_DIR/pyarrow-test-$PYTHON_VERSION conda create -y -q -p $CONDA_ENV_DIR python=$PYTHON_VERSION cmake curl conda activate $CONDA_ENV_DIR +# We should use zlib in the target Python directory to avoid loading +# wrong libpython on macOS at run-time. If we use zlib in +# $ARROW_BUILD_TOOLCHAIN and libpython3.6m.dylib exists in both +# $ARROW_BUILD_TOOLCHAIN and $CONDA_ENV_DIR, python-test uses +# libpython3.6m.dylib on $ARROW_BUILD_TOOLCHAIN not $CONDA_ENV_DIR. +# libpython3.6m.dylib on $ARROW_BUILD_TOOLCHAIN doesn't have NumPy. So +# python-test fails. +export ZLIB_HOME=$CONDA_ENV_DIR + python --version which python diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 462581c3f47..62415ad1cd6 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -55,6 +55,7 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules") include(CMakeParseArguments) include(ExternalProject) +include(FindPkgConfig) # This ensures that things like gnu++11 get passed correctly set(CMAKE_CXX_STANDARD 11) @@ -264,8 +265,6 @@ Pass multiple labels by dividing with semicolons") "Brotli static lib suffix used on Windows with MSVC (default -static)") set(SNAPPY_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING "Snappy static lib suffix used on Windows with MSVC (default is empty string)") - set(ZLIB_MSVC_STATIC_LIB_SUFFIX "libstatic" CACHE STRING - "Zlib static lib suffix used on Windows with MSVC (default libstatic)") set(LZ4_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING "Lz4 static lib suffix used on Windows with MSVC (default _static)") set(ZSTD_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING @@ -619,6 +618,9 @@ if (ARROW_WITH_SNAPPY) endif() if (ARROW_WITH_ZLIB) + if (ZLIB_SHARED_LIB) + SET(ARROW_LINK_LIBS zlib ${ARROW_LINK_LIBS}) + endif() SET(ARROW_STATIC_LINK_LIBS zlib ${ARROW_STATIC_LINK_LIBS}) endif() diff --git a/cpp/apidoc/Windows.md b/cpp/apidoc/Windows.md index 07c36349154..5199c2fdbfa 100644 --- a/cpp/apidoc/Windows.md +++ b/cpp/apidoc/Windows.md @@ -90,10 +90,6 @@ If you decided to use pre-built 3rd party dependencies libs, it's possible to configure Arrow's cmake build script to search for customized names of 3rd party static libs. -`zlib`. Pass `-DARROW_ZLIB_VENDORED=OFF` to enable lookup of custom zlib -build. Set `ZLIB_HOME` environment variable. Pass -`-DZLIB_MSVC_STATIC_LIB_SUFFIX=%ZLIB_SUFFIX%` to link with z%ZLIB_SUFFIX%.lib - `brotli`. Set `BROTLI_HOME` environment variable. Pass `-DBROTLI_MSVC_STATIC_LIB_SUFFIX=%BROTLI_SUFFIX%` to link with brotli*%BROTLI_SUFFIX%.lib. For brotli versions <= 0.6.0 installed from diff --git a/cpp/cmake_modules/BuildUtils.cmake b/cpp/cmake_modules/BuildUtils.cmake index e0564175e3f..257c7592ae2 100644 --- a/cpp/cmake_modules/BuildUtils.cmake +++ b/cpp/cmake_modules/BuildUtils.cmake @@ -38,7 +38,7 @@ function(ADD_THIRDPARTY_LIB LIB_NAME) SET(AUG_LIB_NAME "${LIB_NAME}_shared") add_library(${AUG_LIB_NAME} SHARED IMPORTED) - if(MSVC) + if(WIN32) # Mark the ”.lib” location as part of a Windows DLL set_target_properties(${AUG_LIB_NAME} PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}") @@ -66,16 +66,18 @@ function(ADD_THIRDPARTY_LIB LIB_NAME) message("Added static library dependency ${LIB_NAME}: ${ARG_STATIC_LIB}") elseif(ARG_SHARED_LIB) add_library(${LIB_NAME} SHARED IMPORTED) - set_target_properties(${LIB_NAME} - PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}") SET(AUG_LIB_NAME "${LIB_NAME}_shared") add_library(${AUG_LIB_NAME} SHARED IMPORTED) - if(MSVC) + if(WIN32) # Mark the ”.lib” location as part of a Windows DLL + set_target_properties(${LIB_NAME} + PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}") set_target_properties(${AUG_LIB_NAME} PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}") else() + set_target_properties(${LIB_NAME} + PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}") set_target_properties(${AUG_LIB_NAME} PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}") endif() diff --git a/cpp/cmake_modules/FindZLIB.cmake b/cpp/cmake_modules/FindZLIB.cmake index 78b84f2073f..ca6060b57cc 100644 --- a/cpp/cmake_modules/FindZLIB.cmake +++ b/cpp/cmake_modules/FindZLIB.cmake @@ -27,8 +27,6 @@ # - Find ZLIB (zlib.h, libz.a, libz.so, and libz.so.1) # This module defines # ZLIB_INCLUDE_DIR, directory containing headers -# ZLIB_LIBS, directory containing zlib libraries -# ZLIB_STATIC_LIB, path to libz.a # ZLIB_SHARED_LIB, path to libz's shared library # ZLIB_FOUND, whether zlib has been found @@ -39,52 +37,54 @@ elseif ( ZLIB_HOME ) list( APPEND _zlib_roots ${ZLIB_HOME} ) endif() -# Try the parameterized roots, if they exist -if ( _zlib_roots ) - find_path( ZLIB_INCLUDE_DIR NAMES zlib.h - PATHS ${_zlib_roots} NO_DEFAULT_PATH - PATH_SUFFIXES "include" ) - find_library( ZLIB_LIBRARIES NAMES libz.a zlib - PATHS ${_zlib_roots} NO_DEFAULT_PATH - PATH_SUFFIXES "lib" ) +if (MSVC) + # zlib uses zlib.lib for Windows. + set(ZLIB_LIB_NAME zlib.lib) else () - find_path( ZLIB_INCLUDE_DIR NAMES zlib.h ) - # Only look for the static library - find_library( ZLIB_LIBRARIES NAMES libz.a zlib ) + # zlib uses libz.so for non Windows. + set(ZLIB_LIB_NAME + ${CMAKE_SHARED_LIBRARY_PREFIX}z${CMAKE_SHARED_LIBRARY_SUFFIX}) endif () +# Try the parameterized roots, if they exist +if (_zlib_roots) + find_path(ZLIB_INCLUDE_DIR NAMES zlib.h + PATHS ${_zlib_roots} NO_DEFAULT_PATH + PATH_SUFFIXES "include") + find_library(ZLIB_SHARED_LIB + NAMES ${ZLIB_LIB_NAME} + PATHS ${_zlib_roots} NO_DEFAULT_PATH + PATH_SUFFIXES "lib") +else () + pkg_check_modules(PKG_ZLIB zlib) + if (PKG_ZLIB_FOUND) + set(ZLIB_INCLUDE_DIR ${PKG_ZLIB_INCLUDEDIR}) + find_library(ZLIB_SHARED_LIB + NAMES ${ZLIB_LIB_NAME} + PATHS ${PKG_ZLIB_LIBDIR} NO_DEFAULT_PATH) + else () + find_path(ZLIB_INCLUDE_DIR NAMES zlib.h) + find_library(ZLIB_SHARED_LIB NAMES ${ZLIB_LIB_NAME}) + endif () +endif () -if (ZLIB_INCLUDE_DIR AND (PARQUET_MINIMAL_DEPENDENCY OR ZLIB_LIBRARIES)) +if (ZLIB_INCLUDE_DIR AND ZLIB_SHARED_LIB) set(ZLIB_FOUND TRUE) - get_filename_component( ZLIB_LIBS ${ZLIB_LIBRARIES} PATH ) - set(ZLIB_HEADER_NAME zlib.h) - set(ZLIB_HEADER ${ZLIB_INCLUDE_DIR}/${ZLIB_HEADER_NAME}) - set(ZLIB_LIB_NAME z) - if (MSVC) - if (NOT ZLIB_MSVC_STATIC_LIB_SUFFIX) - set(ZLIB_MSVC_STATIC_LIB_SUFFIX libstatic) - endif() - set(ZLIB_MSVC_SHARED_LIB_SUFFIX lib) - endif() - set(ZLIB_STATIC_LIB ${ZLIB_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${ZLIB_LIB_NAME}${ZLIB_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}) - set(ZLIB_SHARED_LIB ${ZLIB_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${ZLIB_LIB_NAME}${ZLIB_MSVC_SHARED_LIB_SUFFIX}${CMAKE_SHARED_LIBRARY_SUFFIX}) else () set(ZLIB_FOUND FALSE) endif () if (ZLIB_FOUND) if (NOT ZLIB_FIND_QUIETLY) - if (PARQUET_MINIMAL_DEPENDENCY) - message(STATUS "Found the ZLIB header: ${ZLIB_HEADER}") - else() - message(STATUS "Found the ZLIB library: ${ZLIB_LIBRARIES}") + if (ZLIB_SHARED_LIB) + message(STATUS "Found the ZLIB shared library: ${ZLIB_SHARED_LIB}") endif () endif () else () if (NOT ZLIB_FIND_QUIETLY) set(ZLIB_ERR_MSG "Could not find the ZLIB library. Looked in ") if ( _zlib_roots ) - set(ZLIB_ERR_MSG "${ZLIB_ERR_MSG} in ${_zlib_roots}.") + set(ZLIB_ERR_MSG "${ZLIB_ERR_MSG} ${_zlib_roots}.") else () set(ZLIB_ERR_MSG "${ZLIB_ERR_MSG} system search paths.") endif () @@ -98,8 +98,6 @@ endif () mark_as_advanced( ZLIB_INCLUDE_DIR - ZLIB_LIBS ZLIB_LIBRARIES - ZLIB_STATIC_LIB ZLIB_SHARED_LIB ) diff --git a/cpp/cmake_modules/ThirdpartyToolchain.cmake b/cpp/cmake_modules/ThirdpartyToolchain.cmake index 36dcf3017d7..36f760d21fa 100644 --- a/cpp/cmake_modules/ThirdpartyToolchain.cmake +++ b/cpp/cmake_modules/ThirdpartyToolchain.cmake @@ -720,6 +720,13 @@ if (ARROW_WITH_ZLIB) # ZLIB if("${ZLIB_HOME}" STREQUAL "") + find_package(ZLIB) + else() + find_package(ZLIB REQUIRED) + endif() + if(ZLIB_FOUND) + ADD_THIRDPARTY_LIB(zlib SHARED_LIB ${ZLIB_SHARED_LIB}) + else() set(ZLIB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep/src/zlib_ep-install") set(ZLIB_HOME "${ZLIB_PREFIX}") set(ZLIB_INCLUDE_DIR "${ZLIB_PREFIX}/include") @@ -739,25 +746,18 @@ if (ARROW_WITH_ZLIB) -DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS} -DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS} -DBUILD_SHARED_LIBS=OFF) + ADD_THIRDPARTY_LIB(zlib + STATIC_LIB ${ZLIB_STATIC_LIB}) ExternalProject_Add(zlib_ep URL ${ZLIB_SOURCE_URL} ${EP_LOG_OPTIONS} BUILD_BYPRODUCTS "${ZLIB_STATIC_LIB}" CMAKE_ARGS ${ZLIB_CMAKE_ARGS}) - set(ZLIB_VENDORED 1) - else() - find_package(ZLIB REQUIRED) - set(ZLIB_VENDORED 0) + add_dependencies(zlib zlib_ep) endif() include_directories(SYSTEM ${ZLIB_INCLUDE_DIR}) - ADD_THIRDPARTY_LIB(zlib - STATIC_LIB ${ZLIB_STATIC_LIB}) - - if (ZLIB_VENDORED) - add_dependencies(zlib zlib_ep) - endif() endif() if (ARROW_WITH_SNAPPY) @@ -1200,6 +1200,15 @@ if (NOT THRIFT_FOUND) endif() set(THRIFT_STATIC_LIB "${THRIFT_PREFIX}/lib/${THRIFT_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}") + if (ZLIB_SHARED_LIB) + set(THRIFT_CMAKE_ARGS "-DZLIB_LIBRARY=${ZLIB_SHARED_LIB}" + ${THRIFT_CMAKE_ARGS}) + else() + set(THRIFT_CMAKE_ARGS "-DZLIB_LIBRARY=${ZLIB_STATIC_LIB}" + ${THRIFT_CMAKE_ARGS}) + endif() + set(THRIFT_DEPENDENCIES ${THRIFT_DEPENDENCIES} zlib) + if (MSVC) set(WINFLEXBISON_VERSION 2.4.9) set(WINFLEXBISON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/winflexbison_ep/src/winflexbison_ep-install") @@ -1216,11 +1225,9 @@ if (NOT THRIFT_FOUND) set(THRIFT_CMAKE_ARGS "-DFLEX_EXECUTABLE=${WINFLEXBISON_PREFIX}/win_flex.exe" "-DBISON_EXECUTABLE=${WINFLEXBISON_PREFIX}/win_bison.exe" "-DZLIB_INCLUDE_DIR=${ZLIB_INCLUDE_DIR}" - "-DZLIB_LIBRARY=${ZLIB_STATIC_LIB}" "-DWITH_SHARED_LIB=OFF" "-DWITH_PLUGIN=OFF" ${THRIFT_CMAKE_ARGS}) - set(THRIFT_DEPENDENCIES ${THRIFT_DEPENDENCIES} zlib_ep) elseif (APPLE) if (DEFINED BISON_EXECUTABLE) set(THRIFT_CMAKE_ARGS "-DBISON_EXECUTABLE=${BISON_EXECUTABLE}" diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index f3cd6b378c0..883b2dc04cc 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -263,6 +263,35 @@ function(bundle_boost_lib library_path) endif() endfunction() +function(bundle_zlib) + # We can assume that manylinux1 and macosx have system zlib. + # See https://mail.python.org/mm3/archives/list/distutils-sig@python.org/thread/ZZG6GL3XTBLBJXSITYHEXMFKN43EREB7/ + # for manylinux1. + if (MSVC) + # zlib uses zlib.dll for Windows + set(ZLIB_SHARED_LIB_NAME zlib.dll) + if (DEFINED ENV{ARROW_BUILD_TOOLCHAIN}) + set(ZLIB_HOME "$ENV{ARROW_BUILD_TOOLCHAIN}") + endif() + if (DEFINED ENV{ZLIB_HOME}) + set(ZLIB_HOME "$ENV{ZLIB_HOME}") + endif() + if ("${ZLIB_HOME}" STREQUAL "") + find_library(ZLIB_SHARED_LIB NAMES ${ZLIB_SHARED_LIB_NAME}) + else() + find_library(ZLIB_SHARED_LIB + NAMES ${ZLIB_SHARED_LIB_NAME} + PATHS ${ZLIB_HOME} NO_DEFAULT_PATH + PATH_SUFFIXES "bin") + endif() + if (ZLIB_SHARED_LIB) + file(COPY + ${ZLIB_SHARED_LIB} + DESTINATION ${BUILD_OUTPUT_ROOT_DIRECTORY}) + endif() + endif() +endfunction() + # Always bundle includes file(COPY ${ARROW_INCLUDE_DIR}/arrow DESTINATION ${BUILD_OUTPUT_ROOT_DIRECTORY}/include) @@ -302,6 +331,8 @@ if (PYARROW_BUNDLE_ARROW_CPP) bundle_boost_lib(Boost_SYSTEM_LIBRARY) endif() + bundle_zlib() + if (MSVC) bundle_arrow_implib(ARROW_SHARED_IMP_LIB) bundle_arrow_implib(ARROW_PYTHON_SHARED_IMP_LIB) diff --git a/python/setup.py b/python/setup.py old mode 100644 new mode 100755 index f554fb6419b..a90399893e4 --- a/python/setup.py +++ b/python/setup.py @@ -291,6 +291,10 @@ def _run_cmake(self): move_shared_libs( build_prefix, build_lib, "{}_regex".format(self.boost_namespace)) + if sys.platform == 'win32': + # zlib uses zlib.dll for Windows + zlib_lib_name = 'zlib' + move_shared_libs(build_prefix, build_lib, zlib_lib_name) print('Bundling includes: ' + pjoin(build_prefix, 'include')) if os.path.exists(pjoin(build_lib, 'pyarrow', 'include')):