ARROW-3128: [C++] Support system shared zlib

.travis.yml

@@ -261,8 +261,20 @@ matrix:
 
 
 after_failure:
-- COREFILE=$(find . -maxdepth 2 -name "core*" | head -n 1)
-- if [[ -f "$COREFILE" ]]; then gdb -c "$COREFILE" example -ex "thread apply all bt" -ex "set pagination 0" -batch; fi
+- |
+    if [ "$TRAVIS_OS_NAME" = "osx" ]; then
+      COREFILE=$(find /cores -maxdepth 1 -type f -name "core.*" | head -n 1)
+      if [[ -f "$COREFILE" ]]; then
+        lldb -c "$COREFILE" --batch --one-line "thread backtrace all -e true"
+      fi
+      ls -la ~/Library/Logs/DiagnosticReports/
+      cat ~/Library/Logs/DiagnosticReports/*.crash
+    else
+      COREFILE=$(find . -maxdepth 2 -name "core*" | head -n 1)
+      if [[ -f "$COREFILE" ]]; then
+        gdb -c "$COREFILE" example -ex "thread apply all bt" -ex "set pagination 0" -batch
+      fi
+    fi
 
 env:
   global:

ci/test-cmake-build-script.bat

@@ -39,6 +39,11 @@ cmake -G "%GENERATOR%" ^
 FINDSTR /M /C:"Could not find the Flatbuffers library" error.txt || exit /B
 set FLATBUFFERS_HOME=
 
+popd
+rmdir /S /Q %BUILD_DIR%
+mkdir %BUILD_DIR%
+pushd %BUILD_DIR%
+
 echo Test cmake script errors out on gflags missed
 set GFLAGS_HOME=WrongPath
 
@@ -51,6 +56,11 @@ cmake -G "%GENERATOR%" ^
 FINDSTR /M /C:"No static or shared library provided for gflags" error.txt || exit /B
 set GFLAGS_HOME=
 
+popd
+rmdir /S /Q %BUILD_DIR%
+mkdir %BUILD_DIR%
+pushd %BUILD_DIR%
+
 echo Test cmake script errors out on snappy missed
 set SNAPPY_HOME=WrongPath
 
@@ -63,6 +73,11 @@ cmake -G "%GENERATOR%" ^
 FINDSTR /M /C:"Could not find the Snappy library" error.txt || exit /B
 set SNAPPY_HOME=
 
+popd
+rmdir /S /Q %BUILD_DIR%
+mkdir %BUILD_DIR%
+pushd %BUILD_DIR%
+
 echo Test cmake script errors out on zlib missed
 set ZLIB_HOME=WrongPath
 
@@ -75,6 +90,11 @@ cmake -G "%GENERATOR%" ^
 FINDSTR /M /C:"Could not find the ZLIB library" error.txt || exit /B
 set ZLIB_HOME=
 
+popd
+rmdir /S /Q %BUILD_DIR%
+mkdir %BUILD_DIR%
+pushd %BUILD_DIR%
+
 echo Test cmake script errors out on brotli missed
 set BROTLI_HOME=WrongPath
 
@@ -87,6 +107,11 @@ cmake -G "%GENERATOR%" ^
 FINDSTR /M /C:"Could not find the Brotli library" error.txt || exit /B
 set BROTLI_HOME=
 
+popd
+rmdir /S /Q %BUILD_DIR%
+mkdir %BUILD_DIR%
+pushd %BUILD_DIR%
+
 echo Test cmake script errors out on lz4 missed
 set LZ4_HOME=WrongPath
 
@@ -99,6 +124,11 @@ cmake -G "%GENERATOR%" ^
 FINDSTR /M /C:"No static or shared library provided for lz4_static" error.txt || exit /B
 set LZ4_HOME=
 
+popd
+rmdir /S /Q %BUILD_DIR%
+mkdir %BUILD_DIR%
+pushd %BUILD_DIR%
+
 echo Test cmake script errors out on zstd missed
 set ZSTD_HOME=WrongPath
 
@@ -134,10 +164,15 @@ cmake -G "%GENERATOR%" ^
       .. 2>output.txt
 
 set LIBRARY_FOUND_MSG=Added static library dependency
-for %%x in (snappy gflags zlib brotli_enc brotli_dec brotli_common lz4_static zstd_static) do (
+for %%x in (snappy gflags brotli_enc brotli_dec brotli_common lz4_static zstd_static) do (
+    echo Checking %%x library path
+    FINDSTR /C:"%LIBRARY_FOUND_MSG% %%x: %CONDA_PREFIX:\=/%" output.txt || exit /B
+)
+set LIBRARY_FOUND_MSG=Added shared library dependency
+for %%x in (zlib) do (
     echo Checking %%x library path
     FINDSTR /C:"%LIBRARY_FOUND_MSG% %%x: %CONDA_PREFIX:\=/%" output.txt || exit /B
 )
 
 popd
-rmdir /S /Q %BUILD_DIR%
+rmdir /S /Q %BUILD_DIR%

ci/travis_script_python.sh

@@ -35,6 +35,15 @@ CONDA_ENV_DIR=$TRAVIS_BUILD_DIR/pyarrow-test-$PYTHON_VERSION
 conda create -y -q -p $CONDA_ENV_DIR python=$PYTHON_VERSION cmake curl
 conda activate $CONDA_ENV_DIR
 
+# We should use zlib in the target Python directory to avoid loading
+# wrong libpython on macOS at run-time. If we use zlib in
+# $ARROW_BUILD_TOOLCHAIN and libpython3.6m.dylib exists in both
+# $ARROW_BUILD_TOOLCHAIN and $CONDA_ENV_DIR, python-test uses
+# libpython3.6m.dylib on $ARROW_BUILD_TOOLCHAIN not $CONDA_ENV_DIR.
+# libpython3.6m.dylib on $ARROW_BUILD_TOOLCHAIN doesn't have NumPy. So
+# python-test fails.
+export ZLIB_HOME=$CONDA_ENV_DIR
+
 python --version
 which python
 

cpp/CMakeLists.txt

@@ -55,6 +55,7 @@ set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
 
 include(CMakeParseArguments)
 include(ExternalProject)
+include(FindPkgConfig)
 
 # This ensures that things like gnu++11 get passed correctly
 set(CMAKE_CXX_STANDARD 11)
@@ -264,8 +265,6 @@ Pass multiple labels by dividing with semicolons")
       "Brotli static lib suffix used on Windows with MSVC (default -static)")
     set(SNAPPY_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING
       "Snappy static lib suffix used on Windows with MSVC (default is empty string)")
-    set(ZLIB_MSVC_STATIC_LIB_SUFFIX "libstatic" CACHE STRING
-      "Zlib static lib suffix used on Windows with MSVC (default libstatic)")
     set(LZ4_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING
       "Lz4 static lib suffix used on Windows with MSVC (default _static)")
     set(ZSTD_MSVC_STATIC_LIB_SUFFIX "_static" CACHE STRING
@@ -619,6 +618,9 @@ if (ARROW_WITH_SNAPPY)
 endif()
 
 if (ARROW_WITH_ZLIB)
+  if (ZLIB_SHARED_LIB)
+    SET(ARROW_LINK_LIBS zlib ${ARROW_LINK_LIBS})
+  endif()
   SET(ARROW_STATIC_LINK_LIBS zlib ${ARROW_STATIC_LINK_LIBS})
 endif()
 

cpp/apidoc/Windows.md

@@ -90,10 +90,6 @@ If you decided to use pre-built 3rd party dependencies libs, it's possible to
 configure Arrow's cmake build script to search for customized names of 3rd
 party static libs.
 
-`zlib`. Pass `-DARROW_ZLIB_VENDORED=OFF` to enable lookup of custom zlib
-build. Set `ZLIB_HOME` environment variable. Pass
-`-DZLIB_MSVC_STATIC_LIB_SUFFIX=%ZLIB_SUFFIX%` to link with z%ZLIB_SUFFIX%.lib
-
 `brotli`. Set `BROTLI_HOME` environment variable. Pass
 `-DBROTLI_MSVC_STATIC_LIB_SUFFIX=%BROTLI_SUFFIX%` to link with
 brotli*%BROTLI_SUFFIX%.lib. For brotli versions <= 0.6.0 installed from

cpp/cmake_modules/BuildUtils.cmake

@@ -38,7 +38,7 @@ function(ADD_THIRDPARTY_LIB LIB_NAME)
     SET(AUG_LIB_NAME "${LIB_NAME}_shared")
     add_library(${AUG_LIB_NAME} SHARED IMPORTED)
 
-    if(MSVC)
+    if(WIN32)
         # Mark the ”.lib” location as part of a Windows DLL
         set_target_properties(${AUG_LIB_NAME}
             PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}")
@@ -66,16 +66,18 @@ function(ADD_THIRDPARTY_LIB LIB_NAME)
     message("Added static library dependency ${LIB_NAME}: ${ARG_STATIC_LIB}")
   elseif(ARG_SHARED_LIB)
     add_library(${LIB_NAME} SHARED IMPORTED)
-    set_target_properties(${LIB_NAME}
-      PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
     SET(AUG_LIB_NAME "${LIB_NAME}_shared")
     add_library(${AUG_LIB_NAME} SHARED IMPORTED)
 
-    if(MSVC)
+    if(WIN32)
         # Mark the ”.lib” location as part of a Windows DLL
+        set_target_properties(${LIB_NAME}
+            PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}")
         set_target_properties(${AUG_LIB_NAME}
             PROPERTIES IMPORTED_IMPLIB "${ARG_SHARED_LIB}")
     else()
+        set_target_properties(${LIB_NAME}
+            PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
         set_target_properties(${AUG_LIB_NAME}
             PROPERTIES IMPORTED_LOCATION "${ARG_SHARED_LIB}")
     endif()

cpp/cmake_modules/FindZLIB.cmake

@@ -27,8 +27,6 @@
 # - Find ZLIB (zlib.h, libz.a, libz.so, and libz.so.1)
 # This module defines
 #  ZLIB_INCLUDE_DIR, directory containing headers
-#  ZLIB_LIBS, directory containing zlib libraries
-#  ZLIB_STATIC_LIB, path to libz.a
 #  ZLIB_SHARED_LIB, path to libz's shared library
 #  ZLIB_FOUND, whether zlib has been found
 
@@ -39,52 +37,54 @@ elseif ( ZLIB_HOME )
     list( APPEND _zlib_roots ${ZLIB_HOME} )
 endif()
 
-# Try the parameterized roots, if they exist
-if ( _zlib_roots )
-    find_path( ZLIB_INCLUDE_DIR NAMES zlib.h
-        PATHS ${_zlib_roots} NO_DEFAULT_PATH
-        PATH_SUFFIXES "include" )
-    find_library( ZLIB_LIBRARIES NAMES libz.a zlib
-        PATHS ${_zlib_roots} NO_DEFAULT_PATH
-        PATH_SUFFIXES "lib" )
+if (MSVC)
+  # zlib uses zlib.lib for Windows.
+  set(ZLIB_LIB_NAME zlib.lib)
 else ()
-    find_path( ZLIB_INCLUDE_DIR NAMES zlib.h )
-    # Only look for the static library
-    find_library( ZLIB_LIBRARIES NAMES libz.a zlib )
+  # zlib uses libz.so for non Windows.
+  set(ZLIB_LIB_NAME
+    ${CMAKE_SHARED_LIBRARY_PREFIX}z${CMAKE_SHARED_LIBRARY_SUFFIX})
 endif ()
 
+# Try the parameterized roots, if they exist
+if (_zlib_roots)
+  find_path(ZLIB_INCLUDE_DIR NAMES zlib.h
+    PATHS ${_zlib_roots} NO_DEFAULT_PATH
+    PATH_SUFFIXES "include")
+  find_library(ZLIB_SHARED_LIB
+    NAMES ${ZLIB_LIB_NAME}
+    PATHS ${_zlib_roots} NO_DEFAULT_PATH
+    PATH_SUFFIXES "lib")
+else ()
+  pkg_check_modules(PKG_ZLIB zlib)
+  if (PKG_ZLIB_FOUND)
+    set(ZLIB_INCLUDE_DIR ${PKG_ZLIB_INCLUDEDIR})
+    find_library(ZLIB_SHARED_LIB
+      NAMES ${ZLIB_LIB_NAME}
+      PATHS ${PKG_ZLIB_LIBDIR} NO_DEFAULT_PATH)
+  else ()
+    find_path(ZLIB_INCLUDE_DIR NAMES zlib.h)
+    find_library(ZLIB_SHARED_LIB NAMES ${ZLIB_LIB_NAME})
+  endif ()
+endif ()
 
-if (ZLIB_INCLUDE_DIR AND (PARQUET_MINIMAL_DEPENDENCY OR ZLIB_LIBRARIES))
+if (ZLIB_INCLUDE_DIR AND ZLIB_SHARED_LIB)
   set(ZLIB_FOUND TRUE)
-  get_filename_component( ZLIB_LIBS ${ZLIB_LIBRARIES} PATH )
-  set(ZLIB_HEADER_NAME zlib.h)
-  set(ZLIB_HEADER ${ZLIB_INCLUDE_DIR}/${ZLIB_HEADER_NAME})
-  set(ZLIB_LIB_NAME z)
-  if (MSVC)
-    if (NOT ZLIB_MSVC_STATIC_LIB_SUFFIX)
-      set(ZLIB_MSVC_STATIC_LIB_SUFFIX libstatic)
-    endif()
-    set(ZLIB_MSVC_SHARED_LIB_SUFFIX lib)
-  endif()
-  set(ZLIB_STATIC_LIB ${ZLIB_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${ZLIB_LIB_NAME}${ZLIB_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
-  set(ZLIB_SHARED_LIB ${ZLIB_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${ZLIB_LIB_NAME}${ZLIB_MSVC_SHARED_LIB_SUFFIX}${CMAKE_SHARED_LIBRARY_SUFFIX})
 else ()
   set(ZLIB_FOUND FALSE)
 endif ()
 
 if (ZLIB_FOUND)
   if (NOT ZLIB_FIND_QUIETLY)
-    if (PARQUET_MINIMAL_DEPENDENCY)
-      message(STATUS "Found the ZLIB header: ${ZLIB_HEADER}")
-    else()
-      message(STATUS "Found the ZLIB library: ${ZLIB_LIBRARIES}")
+    if (ZLIB_SHARED_LIB)
+      message(STATUS "Found the ZLIB shared library: ${ZLIB_SHARED_LIB}")
     endif ()
   endif ()
 else ()
   if (NOT ZLIB_FIND_QUIETLY)
     set(ZLIB_ERR_MSG "Could not find the ZLIB library. Looked in ")
     if ( _zlib_roots )
-      set(ZLIB_ERR_MSG "${ZLIB_ERR_MSG} in ${_zlib_roots}.")
+      set(ZLIB_ERR_MSG "${ZLIB_ERR_MSG} ${_zlib_roots}.")
     else ()
       set(ZLIB_ERR_MSG "${ZLIB_ERR_MSG} system search paths.")
     endif ()
@@ -98,8 +98,6 @@ endif ()
 
 mark_as_advanced(
   ZLIB_INCLUDE_DIR
-  ZLIB_LIBS
   ZLIB_LIBRARIES
-  ZLIB_STATIC_LIB
   ZLIB_SHARED_LIB
 )

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -720,6 +720,13 @@ if (ARROW_WITH_ZLIB)
 # ZLIB
 
   if("${ZLIB_HOME}" STREQUAL "")
+    find_package(ZLIB)
+  else()
+    find_package(ZLIB REQUIRED)
+  endif()
+  if(ZLIB_FOUND)
+    ADD_THIRDPARTY_LIB(zlib SHARED_LIB ${ZLIB_SHARED_LIB})
+  else()
     set(ZLIB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep/src/zlib_ep-install")
     set(ZLIB_HOME "${ZLIB_PREFIX}")
     set(ZLIB_INCLUDE_DIR "${ZLIB_PREFIX}/include")
@@ -739,25 +746,18 @@ if (ARROW_WITH_ZLIB)
                         -DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS}
                         -DCMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_C_FLAGS}
                         -DBUILD_SHARED_LIBS=OFF)
+    ADD_THIRDPARTY_LIB(zlib
+      STATIC_LIB ${ZLIB_STATIC_LIB})
 
     ExternalProject_Add(zlib_ep
       URL ${ZLIB_SOURCE_URL}
       ${EP_LOG_OPTIONS}
       BUILD_BYPRODUCTS "${ZLIB_STATIC_LIB}"
       CMAKE_ARGS ${ZLIB_CMAKE_ARGS})
-    set(ZLIB_VENDORED 1)
-  else()
-    find_package(ZLIB REQUIRED)
-    set(ZLIB_VENDORED 0)
+    add_dependencies(zlib zlib_ep)
   endif()
 
   include_directories(SYSTEM ${ZLIB_INCLUDE_DIR})
-  ADD_THIRDPARTY_LIB(zlib
-    STATIC_LIB ${ZLIB_STATIC_LIB})
-
-  if (ZLIB_VENDORED)
-    add_dependencies(zlib zlib_ep)
-  endif()
 endif()
 
 if (ARROW_WITH_SNAPPY)
@@ -1200,6 +1200,15 @@ if (NOT THRIFT_FOUND)
   endif()
   set(THRIFT_STATIC_LIB "${THRIFT_PREFIX}/lib/${THRIFT_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}")
 
+  if (ZLIB_SHARED_LIB)
+    set(THRIFT_CMAKE_ARGS "-DZLIB_LIBRARY=${ZLIB_SHARED_LIB}"
+                          ${THRIFT_CMAKE_ARGS})
+  else()
+    set(THRIFT_CMAKE_ARGS "-DZLIB_LIBRARY=${ZLIB_STATIC_LIB}"
+                          ${THRIFT_CMAKE_ARGS})
+  endif()
+  set(THRIFT_DEPENDENCIES ${THRIFT_DEPENDENCIES} zlib)
+
   if (MSVC)
     set(WINFLEXBISON_VERSION 2.4.9)
     set(WINFLEXBISON_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/winflexbison_ep/src/winflexbison_ep-install")
@@ -1216,11 +1225,9 @@ if (NOT THRIFT_FOUND)
     set(THRIFT_CMAKE_ARGS "-DFLEX_EXECUTABLE=${WINFLEXBISON_PREFIX}/win_flex.exe"
                           "-DBISON_EXECUTABLE=${WINFLEXBISON_PREFIX}/win_bison.exe"
                           "-DZLIB_INCLUDE_DIR=${ZLIB_INCLUDE_DIR}"
-                          "-DZLIB_LIBRARY=${ZLIB_STATIC_LIB}"
                           "-DWITH_SHARED_LIB=OFF"
                           "-DWITH_PLUGIN=OFF"
                           ${THRIFT_CMAKE_ARGS})
-    set(THRIFT_DEPENDENCIES ${THRIFT_DEPENDENCIES} zlib_ep)
   elseif (APPLE)
     if (DEFINED BISON_EXECUTABLE)
       set(THRIFT_CMAKE_ARGS "-DBISON_EXECUTABLE=${BISON_EXECUTABLE}"