ARROW-4446: [C++][Python] Run Gandiva C++ unit tests in Appveyor, get…

… build and tests working in Python

Resolves

* ARROW-4014 Fix warning about LIBCMT in MSVC build
* ARROW-4445 Run Gandiva C++ tests in Appveyor
* ARROW-4446 Run Gandiva Python unit tests in Appveyor
* ARROW-4404 Appveyor toolchain build does not actually build the project

Author: Wes McKinney <wesm+git@apache.org>

Closes #3567 from wesm/gandiva-python-windows and squashes the following commits:

d9e4b1b <Wes McKinney> Only use /nodefaultlib:libcmt if arrow_use_static_crt=OFF
a8f960d <Wes McKinney> Fix appyveyor.yml
72c97e2 <Wes McKinney> Re-enable all Appyveyor entries
a2e57d1 <Wes McKinney> Prevent move_shared_libs from clobbering the gandiva.cpp file which downstream code is expecting to find
d78ae13 <Wes McKinney> Debug prints
ac402f0 <Wes McKinney> Set language_level = 3
8ae63f1 <Wes McKinney> Fix LLVM 7 issues with MSVC, enable FindClangTools.cmake to find clang-format in conda env
3677115 <Wes McKinney> Do not do verbose linking
773c731 <Wes McKinney> Fixes
00d4126 <Wes McKinney> Disable failing Windows tests
063543c <Wes McKinney> Don't touch linker flags unless on MSVC
e19b688 <Wes McKinney> Upgrade to LLVM 7
707bc49 <Wes McKinney> Fix for gflags
5897045 <Wes McKinney> Retool FindGFlags.cmake a bit to try to fix Windows toolchain build
c2bb420 <Wes McKinney> Install all conda toolchain packages in one go
c285776 <Wes McKinney> Rearrange conda calls to see if fixes build
469a2df <Wes McKinney> Enable gandiva is one build entry
b05cf08 <Wes McKinney> Get Python build and tests for Gandiva working on, add to Appveyor build

appveyor.yml

@@ -48,6 +48,7 @@ environment:
   global:
     USE_CLCACHE: true
     ARROW_BUILD_GANDIVA: "OFF"
+    ARROW_LLVM_VERSION: "7.0.*"
     PYTHON: "3.6"
     ARCH: "64"
 
@@ -61,6 +62,7 @@ environment:
     - JOB: "Toolchain"
       GENERATOR: Visual Studio 14 2015 Win64
       CONFIGURATION: "Release"
+      ARROW_BUILD_GANDIVA: "ON"
     - JOB: "Static_Crt_Build"
       GENERATOR: Ninja
     - JOB: "Build_Debug"

ci/appveyor-cpp-build.bat

@@ -95,33 +95,24 @@ if "%JOB%" == "Build_Debug" (
   exit /B 0
 )
 
-conda create -n arrow -q -y -c conda-forge ^
-      --file=ci\conda_env_python.yml ^
-      python=%PYTHON% ^
-      numpy=1.14 ^
-      thrift-cpp=0.11 ^
-      boost-cpp
-
-call activate arrow
-
-set ARROW_LLVM_VERSION=6.0.1
+set CONDA_PACKAGES=--file=ci\conda_env_python.yml python=%PYTHON% numpy=1.14 thrift-cpp=0.11 boost-cpp
 
 if "%ARROW_BUILD_GANDIVA%" == "ON" (
   @rem Install llvmdev in the toolchain if building gandiva.dll
-  conda install -q -y llvmdev=%ARROW_LLVM_VERSION% || exit /B
+  set CONDA_PACKAGES=%CONDA_PACKAGES% llvmdev=%ARROW_LLVM_VERSION% clangdev=%ARROW_LLVM_VERSION%
 )
 
-@rem Use Boost from Anaconda
-set BOOST_ROOT=%CONDA_PREFIX%\Library
-set BOOST_LIBRARYDIR=%CONDA_PREFIX%\Library\lib
-
 if "%JOB%" == "Toolchain" (
   @rem Install pre-built "toolchain" packages for faster builds
-  conda install -q -y -c conda-forge ^
-        --file=ci\conda_env_cpp.yml ^
-        python=%PYTHON%
-
-  set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
+  set CONDA_PACKAGES=%CONDA_PACKAGES% --file=ci\conda_env_cpp.yml
 )
 
+conda create -n arrow -q -y %CONDA_PACKAGES% -c conda-forge
+
+call activate arrow
+
+@rem Use Boost from Anaconda
+set BOOST_ROOT=%CONDA_PREFIX%\Library
+set BOOST_LIBRARYDIR=%CONDA_PREFIX%\Library\lib
+
 call ci\cpp-msvc-build-main.bat

ci/cpp-msvc-build-main.bat

@@ -22,7 +22,8 @@ set ARROW_HOME=%CONDA_PREFIX%\Library
 set CMAKE_ARGS=-DARROW_VERBOSE_THIRDPARTY_BUILD=OFF
 
 if "%JOB%" == "Toolchain" (
-    set CMAKE_ARGS=%CMAKE_ARGS% -DARROW_WITH_BZ2=ON
+  set CMAKE_ARGS=%CMAKE_ARGS% -DARROW_WITH_BZ2=ON
+  set ARROW_BUILD_TOOLCHAIN=%CONDA_PREFIX%\Library
 )
 
 @rem Retrieve git submodules, configure env var for Parquet unit tests
@@ -85,6 +86,7 @@ set PYARROW_BUNDLE_ARROW_CPP=ON
 set PYARROW_BUNDLE_BOOST=OFF
 set PYARROW_WITH_STATIC_BOOST=ON
 set PYARROW_WITH_PARQUET=ON
+set PYARROW_WITH_GANDIVA=%ARROW_BUILD_GANDIVA%
 set PYARROW_PARALLEL=2
 
 @rem ARROW-3075; pkgconfig is broken for Parquet for now

cpp/CMakeLists.txt

@@ -311,6 +311,10 @@ Note that this requires linking Boost statically"
   # Windows options
 
   if (MSVC)
+    option(MSVC_LINK_VERBOSE
+      "Pass verbose linking options when linking libraries and executables"
+      OFF)
+
     option(ARROW_USE_CLCACHE
       "Use clcache if available"
       ON)

cpp/cmake_modules/FindClangTools.cmake

@@ -53,8 +53,10 @@ set(CLANG_TOOLS_SEARCH_PATHS
   ${ClangTools_PATH}
   $ENV{CLANG_TOOLS_PATH}
   /usr/local/bin /usr/bin
-  "C:/Program Files/LLVM/bin"
-  "${HOMEBREW_PREFIX}/bin")
+  "C:/Program Files/LLVM/bin"  # Windows, non-conda
+  "$ENV{CONDA_PREFIX}/Library/bin"  # Windows, conda
+  "${HOMEBREW_PREFIX}/bin"
+)
 
 find_program(CLANG_TIDY_BIN
   NAMES clang-tidy-${ARROW_LLVM_VERSION}

cpp/cmake_modules/FindGFlags.cmake

@@ -39,16 +39,18 @@ set(GFLAGS_STATIC_LIB_SUFFIX
 set(GFLAGS_STATIC_LIB_NAME
   ${CMAKE_STATIC_LIBRARY_PREFIX}gflags${GFLAGS_STATIC_LIB_SUFFIX})
 
+message(STATUS "GFLAGS_HOME: ${GFLAGS_HOME}")
+
 if ( _gflags_roots )
   find_path(GFLAGS_INCLUDE_DIR NAMES gflags/gflags.h
     PATHS ${_gflags_roots}
     NO_DEFAULT_PATH
     PATH_SUFFIXES "include" )
-  find_library(GFLAGS_SHARED_LIB NAMES gflags
+  find_library(GFLAGS_STATIC_LIB NAMES ${GFLAGS_STATIC_LIB_NAME}
     PATHS ${_gflags_roots}
     NO_DEFAULT_PATH
     PATH_SUFFIXES "lib" )
-  find_library(GFLAGS_STATIC_LIB NAMES ${GFLAGS_STATIC_LIB_NAME}
+  find_library(GFLAGS_SHARED_LIB NAMES gflags
     PATHS ${_gflags_roots}
     NO_DEFAULT_PATH
     PATH_SUFFIXES "lib" )
@@ -57,14 +59,21 @@ else()
     # make sure we don't accidentally pick up a different version
     NO_CMAKE_SYSTEM_PATH
     NO_SYSTEM_ENVIRONMENT_PATH)
-  find_library(GFLAGS_SHARED_LIB gflags
+  find_library(GFLAGS_STATIC_LIB ${GFLAGS_STATIC_LIB_NAME}
     NO_CMAKE_SYSTEM_PATH
     NO_SYSTEM_ENVIRONMENT_PATH)
-  find_library(GFLAGS_STATIC_LIB ${GFLAGS_STATIC_LIB_NAME}
+  find_library(GFLAGS_SHARED_LIB gflags
     NO_CMAKE_SYSTEM_PATH
     NO_SYSTEM_ENVIRONMENT_PATH)
 endif()
 
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(GFLAGS REQUIRED_VARS
-  GFLAGS_SHARED_LIB GFLAGS_STATIC_LIB GFLAGS_INCLUDE_DIR)
+if (GFLAGS_INCLUDE_DIR AND GFLAGS_STATIC_LIB)
+  set(GFLAGS_FOUND TRUE)
+else ()
+  set(GFLAGS_FOUND FALSE)
+endif ()
+
+mark_as_advanced(
+  GFLAGS_INCLUDE_DIR
+  GFLAGS_STATIC_LIB
+)

cpp/cmake_modules/FindGandiva.cmake

@@ -45,9 +45,7 @@ else()
     ${GANDIVA_HOME}/include
     )
 
-  set(GANDIVA_SEARCH_LIB_PATH
-    ${GANDIVA_HOME}/lib
-    )
+  set(GANDIVA_SEARCH_LIB_PATH "${GANDIVA_HOME}")
 
   find_path(GANDIVA_INCLUDE_DIR gandiva/expression_registry.h PATHS
     ${GANDIVA_SEARCH_HEADER_PATHS}
@@ -57,18 +55,30 @@ else()
 endif()
 
 find_library(GANDIVA_LIB_PATH NAMES gandiva
-  PATHS
-  ${GANDIVA_SEARCH_LIB_PATH}
-  NO_DEFAULT_PATH)
+  PATHS ${GANDIVA_SEARCH_LIB_PATH}
+  NO_DEFAULT_PATH
+  PATH_SUFFIXES "lib")
 get_filename_component(GANDIVA_LIBS ${GANDIVA_LIB_PATH} DIRECTORY)
 
-if (GANDIVA_INCLUDE_DIR AND GANDIVA_LIBS)
+find_library(GANDIVA_SHARED_LIB_PATH NAMES gandiva
+  PATHS ${GANDIVA_SEARCH_LIB_PATH}
+  NO_DEFAULT_PATH
+  PATH_SUFFIXES "bin" )
+
+get_filename_component(GANDIVA_SHARED_LIBS ${GANDIVA_SHARED_LIB_PATH} PATH )
+
+if (GANDIVA_INCLUDE_DIR AND GANDIVA_LIBS AND GANDIVA_SHARED_LIBS)
   set(GANDIVA_FOUND TRUE)
   set(GANDIVA_LIB_NAME gandiva)
 
-  set(GANDIVA_STATIC_LIB ${GANDIVA_LIBS}/lib${GANDIVA_LIB_NAME}.a)
-
-  set(GANDIVA_SHARED_LIB ${GANDIVA_LIBS}/lib${GANDIVA_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+  if (MSVC)
+    set(GANDIVA_STATIC_LIB "${GANDIVA_LIBS}/${GANDIVA_LIB_NAME}${GANDIVA_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}")
+    set(GANDIVA_SHARED_LIB "${GANDIVA_SHARED_LIBS}/${GANDIVA_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}")
+    set(GANDIVA_SHARED_IMP_LIB "${GANDIVA_LIBS}/${GANDIVA_LIB_NAME}.lib")
+  else()
+    set(GANDIVA_STATIC_LIB ${GANDIVA_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${GANDIVA_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX})
+    set(GANDIVA_SHARED_LIB ${GANDIVA_SHARED_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${GANDIVA_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+  endif()
 endif()
 
 if (GANDIVA_FOUND)

cpp/cmake_modules/FindParquet.cmake

@@ -30,6 +30,10 @@ if(NOT "$ENV{PARQUET_HOME}" STREQUAL "")
     set(PARQUET_HOME "$ENV{PARQUET_HOME}")
 endif()
 
+if(NOT "$ENV{ARROW_HOME}" STREQUAL "")
+    set(ARROW_HOME "$ENV{ARROW_HOME}")
+endif()
+
 if (MSVC)
   SET(CMAKE_FIND_LIBRARY_SUFFIXES ".lib" ".dll")
 
@@ -38,7 +42,8 @@ if (MSVC)
   endif()
 
   find_library(PARQUET_SHARED_LIBRARIES NAMES parquet
-    PATHS ${PARQUET_HOME} NO_DEFAULT_PATH
+    PATHS ${PARQUET_HOME} ${ARROW_HOME}
+    NO_DEFAULT_PATH
     PATH_SUFFIXES "bin" )
 
   get_filename_component(PARQUET_SHARED_LIBS ${PARQUET_SHARED_LIBRARIES} PATH )

cpp/cmake_modules/SetupCxxFlags.cmake

@@ -411,3 +411,22 @@ else()
 endif()
 
 message(STATUS "Build Type: ${CMAKE_BUILD_TYPE}")
+
+# ----------------------------------------------------------------------
+# MSVC-specific linker options
+
+if (MSVC)
+  set(MSVC_LINKER_FLAGS)
+  if (MSVC_LINK_VERBOSE)
+    set(MSVC_LINKER_FLAGS "${MSVC_LINKER_FLAGS} /VERBOSE:LIB")
+  endif()
+  if (NOT ARROW_USE_STATIC_CRT)
+    set(MSVC_LINKER_FLAGS "${MSVC_LINKER_FLAGS} /NODEFAULTLIB:LIBCMT")
+    set(CMAKE_EXE_LINKER_FLAGS
+      "${CMAKE_EXE_LINKER_FLAGS} ${MSVC_LINKER_FLAGS}")
+    set(CMAKE_MODULE_LINKER_FLAGS
+      "${CMAKE_MODULE_LINKER_FLAGS} ${MSVC_LINKER_FLAGS}")
+    set(CMAKE_SHARED_LINKER_FLAGS
+      "${CMAKE_SHARED_LINKER_FLAGS} ${MSVC_LINKER_FLAGS}")
+  endif()
+endif()

cpp/src/gandiva/engine.cc

@@ -15,6 +15,12 @@
 // specific language governing permissions and limitations
 // under the License.
 
+// TODO(wesm): LLVM 7 produces pesky C4244 that disable pragmas around the LLVM
+// includes seem to not fix as with LLVM 6
+#if defined(_MSC_VER)
+#pragma warning(disable : 4244)
+#endif
+
 #include "gandiva/engine.h"
 
 #include <iostream>

cpp/src/gandiva/expression_registry.h

@@ -63,6 +63,7 @@ class GANDIVA_EXPORT ExpressionRegistry {
   std::unique_ptr<FunctionRegistry> function_registry_;
 };
 
+GANDIVA_EXPORT
 std::vector<std::shared_ptr<FunctionSignature>> GetRegisteredFunctionSignatures();
 
 }  // namespace gandiva

cpp/src/gandiva/precompiled/time_test.cc

@@ -67,13 +67,27 @@ TEST(TestTime, TestCastTimestamp) {
             -1187308799080);
   EXPECT_EQ(castTIMESTAMP_utf8(context_ptr, "1857-02-11 20:31:40.920 -05:30", 30),
             -3562264699080);
+}
+
+#ifndef _WIN32
+
+// TODO(wesm): ARROW-4495. Need to address TZ database issues on Windows
+
+TEST(TestTime, TestCastTimestampWithTZ) {
+  ExecutionContext context;
+  int64_t context_ptr = reinterpret_cast<int64_t>(&context);
 
   EXPECT_EQ(castTIMESTAMP_utf8(context_ptr, "2000-09-23 9:45:30.920 Canada/Pacific", 37),
             969727530920);
   EXPECT_EQ(castTIMESTAMP_utf8(context_ptr, "2012-02-28 23:30:59 Asia/Kolkata", 32),
             1330452059000);
   EXPECT_EQ(castTIMESTAMP_utf8(context_ptr, "1923-10-07 03:03:03 America/New_York", 36),
             -1459094217000);
+}
+
+TEST(TestTime, TestCastTimestampErrors) {
+  ExecutionContext context;
+  int64_t context_ptr = reinterpret_cast<int64_t>(&context);
 
   // error cases
   EXPECT_EQ(castTIMESTAMP_utf8(context_ptr, "20000923", 8), 0);
@@ -93,6 +107,8 @@ TEST(TestTime, TestCastTimestamp) {
   context.Reset();
 }
 
+#endif
+
 TEST(TestTime, TestExtractTime) {
   // 10:20:33
   int32 time_as_millis_in_day = 37233000;

docs/source/python/development.rst

@@ -325,6 +325,8 @@ Now, we build and install Arrow C++ libraries
    cmake --build . --target INSTALL --config Release
    cd ..\..
 
+If building with LLVM, also add `-DARROW_GANDIVA=ON`.
+
 After that, we must put the install directory's bin path in our ``%PATH%``:
 
 .. code-block:: shell

python/CMakeLists.txt

@@ -517,12 +517,23 @@ if (PYARROW_BUILD_GANDIVA)
     bundle_arrow_lib(GANDIVA_SHARED_LIB
       ABI_VERSION ${ARROW_ABI_VERSION}
       SO_VERSION ${ARROW_SO_VERSION})
+
+    if (MSVC)
+      bundle_arrow_implib(GANDIVA_SHARED_IMP_LIB)
+    endif()
+  endif()
+
+  if (MSVC)
+    ADD_THIRDPARTY_LIB(gandiva
+      SHARED_LIB ${GANDIVA_SHARED_IMP_LIB})
+  else()
+    ADD_THIRDPARTY_LIB(gandiva
+      SHARED_LIB ${GANDIVA_SHARED_LIB})
   endif()
 
   set(LINK_LIBS
     ${LINK_LIBS}
-    ${GANDIVA_SHARED_LIB})
-
+    gandiva_shared)
   set(CYTHON_EXTENSIONS ${CYTHON_EXTENSIONS} gandiva)
 endif()
 

python/pyarrow/gandiva.pyx

@@ -18,6 +18,7 @@
 # cython: profile=False
 # distutils: language = c++
 # cython: embedsignature = True
+# cython: language_level = 3
 
 from libcpp cimport bool as c_bool, nullptr
 from libcpp.memory cimport shared_ptr, unique_ptr, make_shared