ARROW-3279: [C++] Allow linking Arrow tests dynamically on Windows

ci/cpp-python-msvc-build.bat

@@ -18,12 +18,18 @@
 @echo on
 
 if "%JOB%" == "Static_Crt_Build" (
+  @rem Since we link the CRT statically, we should also disable building
+  @rem the Arrow shared library to link the tests statically, otherwise
+  @rem the Arrow DLL and the tests end up using a different instance of
+  @rem the CRT, which wreaks havoc.
+
   mkdir cpp\build-debug
   pushd cpp\build-debug
 
   cmake -G "%GENERATOR%" ^
         -DARROW_USE_STATIC_CRT=ON ^
         -DARROW_BOOST_USE_SHARED=OFF ^
+        -DARROW_BUILD_SHARED=OFF ^
         -DCMAKE_BUILD_TYPE=Debug ^
         -DARROW_CXXFLAGS="/MP" ^
         ..  || exit /B
@@ -38,6 +44,7 @@ if "%JOB%" == "Static_Crt_Build" (
   cmake -G "%GENERATOR%" ^
         -DARROW_USE_STATIC_CRT=ON ^
         -DARROW_BOOST_USE_SHARED=OFF ^
+        -DARROW_BUILD_SHARED=OFF ^
         -DCMAKE_BUILD_TYPE=Release ^
         -DARROW_CXXFLAGS="/WX /MP" ^
         ..  || exit /B
@@ -50,13 +57,18 @@ if "%JOB%" == "Static_Crt_Build" (
   exit /B 0
 )
 
+@rem In the configurations below we disable building the Arrow static library
+@rem to save some time.  Unfortunately this will still build the Parquet static
+@rem library because of PARQUET-1420 (Thrift-generated symbols not exported in DLL).
+
 if "%JOB%" == "Build_Debug" (
   mkdir cpp\build-debug
   pushd cpp\build-debug
 
   cmake -G "%GENERATOR%" ^
         -DARROW_BOOST_USE_SHARED=OFF ^
         -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
+        -DARROW_BUILD_STATIC=OFF ^
         -DARROW_CXXFLAGS="/MP" ^
         ..  || exit /B
 
@@ -111,6 +123,7 @@ cmake -G "%GENERATOR%" ^
       -DCMAKE_INSTALL_PREFIX=%CONDA_PREFIX%\Library ^
       -DARROW_BOOST_USE_SHARED=OFF ^
       -DCMAKE_BUILD_TYPE=%CONFIGURATION% ^
+      -DARROW_BUILD_STATIC=OFF ^
       -DARROW_CXXFLAGS="/WX /MP" ^
       -DARROW_PARQUET=ON ^
       -DARROW_PYTHON=ON ^

cpp/CMakeLists.txt

@@ -295,7 +295,9 @@ Always OFF if building binaries"
 endif()
 
 if(ARROW_BUILD_TESTS OR ARROW_BUILD_BENCHMARKS)
-  set(ARROW_BUILD_STATIC ON)
+  if (NOT MSVC AND NOT ARROW_BUILD_STATIC)
+    message(FATAL_ERROR "ARROW_BUILD_TESTS needs ARROW_BUILD_STATIC")
+  endif ()
   set(ARROW_WITH_BROTLI ON)
   set(ARROW_WITH_LZ4 ON)
   set(ARROW_WITH_SNAPPY ON)
@@ -674,11 +676,22 @@ if (NOT MSVC)
     ${CMAKE_DL_LIBS})
 endif()
 
-set(ARROW_MIN_TEST_LIBS
-  arrow_static
-  ${ARROW_STATIC_LINK_LIBS}
-  gtest_main
-  gtest)
+if (ARROW_BUILD_STATIC)
+  set(ARROW_MIN_TEST_LIBS
+    arrow_static
+    ${ARROW_STATIC_LINK_LIBS}
+    gtest_main
+    gtest)
+else ()
+  set(ARROW_MIN_TEST_LIBS
+    arrow_shared
+    ${ARROW_LINK_LIBS}
+    ${BOOST_SYSTEM_LIBRARY}
+    ${BOOST_FILESYSTEM_LIBRARY}
+    ${BOOST_REGEX_LIBRARY}
+    gtest_main
+    gtest)
+endif()
 
 if(NOT MSVC)
   set(ARROW_MIN_TEST_LIBS

cpp/cmake_modules/BuildUtils.cmake

@@ -101,7 +101,7 @@ endfunction()
 
 # \arg OUTPUTS list to append built targets to
 function(ADD_ARROW_LIB LIB_NAME)
-  set(options)
+  set(options BUILD_SHARED BUILD_STATIC)
   set(one_value_args SHARED_LINK_FLAGS)
   set(multi_value_args SOURCES OUTPUTS STATIC_LINK_LIBS STATIC_PRIVATE_LINK_LIBS SHARED_LINK_LIBS SHARED_PRIVATE_LINK_LIBS EXTRA_INCLUDES DEPENDENCIES)
   cmake_parse_arguments(ARG "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN})
@@ -113,14 +113,30 @@ function(ADD_ARROW_LIB LIB_NAME)
     set(${ARG_OUTPUTS})
   endif()
 
+  # Allow overriding ARROW_BUILD_SHARED and ARROW_BUILD_STATIC
+  if (ARG_BUILD_SHARED)
+    set(BUILD_SHARED ${ARG_BUILD_SHARED})
+  else ()
+    set(BUILD_SHARED ${ARROW_BUILD_SHARED})
+  endif()
+  if (ARG_BUILD_STATIC)
+    set(BUILD_STATIC ${ARG_BUILD_STATIC})
+  else ()
+    set(BUILD_STATIC ${ARROW_BUILD_STATIC})
+  endif()
+
   if(MSVC)
+    # MSVC needs to compile C++ separately for each library kind (shared and static)
+    # because of dllexport declarations
     set(LIB_DEPS ${ARG_SOURCES})
     set(EXTRA_DEPS ${ARG_DEPENDENCIES})
 
     if (ARG_EXTRA_INCLUDES)
       set(LIB_INCLUDES ${ARG_EXTRA_INCLUDES})
     endif()
   else()
+    # Otherwise, generate a single "objlib" from all C++ modules and link
+    # that "objlib" into each library kind, to avoid compiling twice
     add_library(${LIB_NAME}_objlib OBJECT
       ${ARG_SOURCES})
     # Necessary to make static linking into other shared libraries work properly
@@ -145,7 +161,7 @@ function(ADD_ARROW_LIB LIB_NAME)
 
   set(RUNTIME_INSTALL_DIR bin)
 
-  if (ARROW_BUILD_SHARED)
+  if (BUILD_SHARED)
     add_library(${LIB_NAME}_shared SHARED ${LIB_DEPS})
     if (EXTRA_DEPS)
       add_dependencies(${LIB_NAME}_shared ${EXTRA_DEPS})
@@ -212,7 +228,7 @@ function(ADD_ARROW_LIB LIB_NAME)
       ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR})
   endif()
 
-  if (ARROW_BUILD_STATIC)
+  if (BUILD_STATIC)
     add_library(${LIB_NAME}_static STATIC ${LIB_DEPS})
     if(EXTRA_DEPS)
       add_dependencies(${LIB_NAME}_static ${EXTRA_DEPS})
@@ -366,7 +382,7 @@ function(ADD_ARROW_TEST REL_TEST_NAME)
     endif()
   endif()
 
-  if(NO_TESTS OR NOT ARROW_BUILD_STATIC)
+  if (NO_TESTS)
     return()
   endif()
   get_filename_component(TEST_NAME ${REL_TEST_NAME} NAME_WE)

cpp/src/arrow/compute/kernels/util-internal.h

@@ -23,6 +23,7 @@
 
 #include "arrow/compute/kernel.h"
 #include "arrow/type_fwd.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
 namespace compute {
@@ -49,18 +50,23 @@ static inline void CopyData(const ArrayData& input, ArrayData* output) {
 
 namespace detail {
 
+ARROW_EXPORT
 Status InvokeUnaryArrayKernel(FunctionContext* ctx, UnaryKernel* kernel,
                               const Datum& value, std::vector<Datum>* outputs);
 
+ARROW_EXPORT
 Status InvokeBinaryArrayKernel(FunctionContext* ctx, BinaryKernel* kernel,
                                const Datum& left, const Datum& right,
                                std::vector<Datum>* outputs);
+ARROW_EXPORT
 Status InvokeBinaryArrayKernel(FunctionContext* ctx, BinaryKernel* kernel,
                                const Datum& left, const Datum& right, Datum* output);
 
+ARROW_EXPORT
 Datum WrapArraysLike(const Datum& value,
                      const std::vector<std::shared_ptr<Array>>& arrays);
 
+ARROW_EXPORT
 Datum WrapDatumsLike(const Datum& value, const std::vector<Datum>& datums);
 
 }  // namespace detail

cpp/src/arrow/io/mman.h

@@ -15,10 +15,11 @@
 
 #endif
 
+#include "arrow/util/windows_compatibility.h"
+
 #include <errno.h>
 #include <io.h>
 #include <sys/types.h>
-#include <windows.h>
 
 #define PROT_NONE 0
 #define PROT_READ 1

cpp/src/arrow/ipc/json-internal.h

@@ -95,21 +95,25 @@ namespace ipc {
 namespace internal {
 namespace json {
 
-Status WriteSchema(const Schema& schema, RjWriter* writer);
-Status WriteRecordBatch(const RecordBatch& batch, RjWriter* writer);
-Status WriteArray(const std::string& name, const Array& array, RjWriter* writer);
+ARROW_EXPORT Status WriteSchema(const Schema& schema, RjWriter* writer);
+ARROW_EXPORT Status WriteRecordBatch(const RecordBatch& batch, RjWriter* writer);
+ARROW_EXPORT Status WriteArray(const std::string& name, const Array& array,
+                               RjWriter* writer);
 
-Status ReadSchema(const rj::Value& json_obj, MemoryPool* pool,
-                  std::shared_ptr<Schema>* schema);
+ARROW_EXPORT Status ReadSchema(const rj::Value& json_obj, MemoryPool* pool,
+                               std::shared_ptr<Schema>* schema);
 
-Status ReadRecordBatch(const rj::Value& json_obj, const std::shared_ptr<Schema>& schema,
-                       MemoryPool* pool, std::shared_ptr<RecordBatch>* batch);
+ARROW_EXPORT Status ReadRecordBatch(const rj::Value& json_obj,
+                                    const std::shared_ptr<Schema>& schema,
+                                    MemoryPool* pool,
+                                    std::shared_ptr<RecordBatch>* batch);
 
-Status ReadArray(MemoryPool* pool, const rj::Value& json_obj,
-                 const std::shared_ptr<DataType>& type, std::shared_ptr<Array>* array);
+ARROW_EXPORT Status ReadArray(MemoryPool* pool, const rj::Value& json_obj,
+                              const std::shared_ptr<DataType>& type,
+                              std::shared_ptr<Array>* array);
 
-Status ReadArray(MemoryPool* pool, const rj::Value& json_obj, const Schema& schema,
-                 std::shared_ptr<Array>* array);
+ARROW_EXPORT Status ReadArray(MemoryPool* pool, const rj::Value& json_obj,
+                              const Schema& schema, std::shared_ptr<Array>* array);
 
 }  // namespace json
 }  // namespace internal

cpp/src/arrow/ipc/json.h

@@ -41,7 +41,7 @@ namespace json {
 /// \brief Write the JSON representation of an Arrow record batch file or stream
 ///
 /// This is used for integration testing
-class JsonWriter {
+class ARROW_EXPORT JsonWriter {
  public:
   ~JsonWriter();
 
@@ -74,7 +74,7 @@ class JsonWriter {
 /// \brief Read the JSON representation of an Arrow record batch file or stream
 ///
 /// This is used for integration testing
-class JsonReader {
+class ARROW_EXPORT JsonReader {
  public:
   ~JsonReader();
 

cpp/src/arrow/python/CMakeLists.txt

@@ -146,7 +146,7 @@ if (ARROW_BUILD_TESTS)
 
   set(ARROW_PYTHON_MIN_TEST_LIBS
     arrow_python_test_main
-    arrow_python_static
+    arrow_python_shared
     arrow_shared)
 
   set(ARROW_PYTHON_TEST_LINK_LIBS ${ARROW_PYTHON_MIN_TEST_LIBS})

cpp/src/arrow/python/decimal.h

@@ -21,6 +21,7 @@
 #include <string>
 
 #include "arrow/type.h"
+#include "arrow/util/visibility.h"
 
 namespace arrow {
 
@@ -37,18 +38,21 @@ class OwnedRef;
 namespace internal {
 
 // \brief Import the Python Decimal type
+ARROW_EXPORT
 Status ImportDecimalType(OwnedRef* decimal_type);
 
 // \brief Convert a Python Decimal object to a C++ string
 // \param[in] python_decimal A Python decimal.Decimal instance
 // \param[out] The string representation of the Python Decimal instance
 // \return The status of the operation
+ARROW_EXPORT
 Status PythonDecimalToString(PyObject* python_decimal, std::string* out);
 
 // \brief Convert a C++ std::string to a Python Decimal instance
 // \param[in] decimal_constructor The decimal type object
 // \param[in] decimal_string A decimal string
 // \return An instance of decimal.Decimal
+ARROW_EXPORT
 PyObject* DecimalFromString(PyObject* decimal_constructor,
                             const std::string& decimal_string);
 
@@ -57,18 +61,21 @@ PyObject* DecimalFromString(PyObject* decimal_constructor,
 // \param[in] arrow_type An instance of arrow::DecimalType
 // \param[out] out A pointer to a Decimal128
 // \return The status of the operation
+ARROW_EXPORT
 Status DecimalFromPythonDecimal(PyObject* python_decimal, const DecimalType& arrow_type,
                                 Decimal128* out);
 
 // \brief Check whether obj is an instance of Decimal
+ARROW_EXPORT
 bool PyDecimal_Check(PyObject* obj);
 
 // \brief Check whether obj is nan. This function will abort the program if the argument
 // is not a Decimal instance
+ARROW_EXPORT
 bool PyDecimal_ISNAN(PyObject* obj);
 
 // \brief Helper class to track and update the precision and scale of a decimal
-class DecimalMetadata {
+class ARROW_EXPORT DecimalMetadata {
  public:
   DecimalMetadata();
   DecimalMetadata(int32_t precision, int32_t scale);

cpp/src/arrow/python/helpers.h

@@ -53,13 +53,15 @@ namespace internal {
 // \brief Import a Python module
 // \param[in] module_name The name of the module
 // \param[out] ref The OwnedRef containing the module PyObject*
+ARROW_EXPORT
 Status ImportModule(const std::string& module_name, OwnedRef* ref);
 
 // \brief Import an object from a Python module
 // \param[in] module A Python module
 // \param[in] name The name of the object to import
 // \param[out] ref The OwnedRef containing the \c name attribute of the Python module \c
 // module
+ARROW_EXPORT
 Status ImportFromModule(const OwnedRef& module, const std::string& name, OwnedRef* ref);
 
 // \brief Check whether obj is an integer, independent of Python versions.
@@ -72,9 +74,11 @@ inline bool IsPyInteger(PyObject* obj) {
 }
 
 // \brief Use pandas missing value semantics to check if a value is null
+ARROW_EXPORT
 bool PandasObjectIsNull(PyObject* obj);
 
 // \brief Check whether obj is a floating-point NaN
+ARROW_EXPORT
 bool PyFloat_IsNaN(PyObject* obj);
 
 inline bool IsPyBinary(PyObject* obj) {
@@ -89,15 +93,19 @@ template <typename Int>
 Status CIntFromPython(PyObject* obj, Int* out, const std::string& overflow_message = "");
 
 // \brief Convert a Python unicode string to a std::string
+ARROW_EXPORT
 Status PyUnicode_AsStdString(PyObject* obj, std::string* out);
 
 // \brief Convert a Python bytes object to a std::string
+ARROW_EXPORT
 std::string PyBytes_AsStdString(PyObject* obj);
 
 // \brief Call str() on the given object and return the result as a std::string
+ARROW_EXPORT
 Status PyObject_StdStringStr(PyObject* obj, std::string* out);
 
 // \brief Return the repr() of the given object (always succeeds)
+ARROW_EXPORT
 std::string PyObject_StdStringRepr(PyObject* obj);
 
 // \brief Cast the given size to int32_t, with error checking
@@ -113,9 +121,12 @@ inline Status CastSize(Py_ssize_t size, int32_t* out,
 
 // \brief Print the Python object's __str__ form along with the passed error
 // message
+ARROW_EXPORT
 Status InvalidValue(PyObject* obj, const std::string& why);
 
+ARROW_EXPORT
 Status IntegerScalarToDoubleSafe(PyObject* obj, double* result);
+ARROW_EXPORT
 Status IntegerScalarToFloat32Safe(PyObject* obj, float* result);
 
 }  // namespace internal