Merge pull request #2717 from sinistersnare/feature/zstd-decompression

Zstd decompression for arrows
finos · Aug 24, 2024 · ca4210f · ca4210f
2 parents e179382 + c084c96
commit ca4210f
Show file tree

Hide file tree

Showing 3 changed files with 26 additions and 7 deletions.
diff --git a/cmake/arrow/CMakeLists.txt b/cmake/arrow/CMakeLists.txt
@@ -90,7 +90,7 @@ set(ARROW_SRCS
     ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/byte_size.cc
     ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/cancel.cc
     ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/compression.cc
-    # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/compression_zstd.cc
+    ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/compression_zstd.cc
     ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/compression_lz4.cc
     ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/counting_semaphore.cc
     ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/debug.cc
@@ -154,11 +154,11 @@ set(ARROW_SRCS
     ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/exec.cc
     ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/expression.cc
     ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/function.cc
-    ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/function_internal.cc    
+    ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/function_internal.cc
     ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernel.cc
     ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/ordering.cc
 
-    
+
     # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/registry.cc
     # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/aggregate_basic.cc
     # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/aggregate_mode.cc
@@ -278,16 +278,19 @@ add_library(arrow STATIC ${ARROW_SRCS})
 
 target_compile_definitions(arrow PUBLIC ARROW_NO_DEPRECATED_API)
 target_compile_definitions(arrow PUBLIC ARROW_STATIC)
-# target_compile_definitions(arrow PUBLIC ARROW_WITH_ZSTD=ON)
+target_compile_definitions(arrow PUBLIC ARROW_WITH_ZSTD=ON)
 target_compile_definitions(arrow PUBLIC ARROW_WITH_LZ4)
 
+target_include_directories(arrow SYSTEM PRIVATE ${zstd_SOURCE_DIR}/lib)
+
 # will need built boost filesystem and system .lib to work, even though
 # perspective itself does not use those dependencies
 target_link_libraries(arrow
     ${double-conversion_LIBRARIES}
     ${Boost_FILESYSTEM_LIBRARY}
     ${Boost_SYSTEM_LIBRARY}
     lz4_static
+    libzstd_static
     ${ARROW_TEST_LINK_TOOLCHAIN})
 
 # find_package(Flatbuffers)

diff --git a/cmake/modules/FindInstallDependency.cmake b/cmake/modules/FindInstallDependency.cmake
@@ -12,7 +12,7 @@ function(psp_build_dep name cmake_file)
     else()
         configure_file(${cmake_file} ${CMAKE_BINARY_DIR}/${name}-download/CMakeLists.txt)
         set(_cwd ${CMAKE_BINARY_DIR}/${name}-download)
-        
+
         message(STATUS "Configuring ${name} in ${_cwd}")
 
         execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" .
@@ -24,7 +24,7 @@ function(psp_build_dep name cmake_file)
         if(result)
             message(FATAL_ERROR "CMake step for ${name} failed:\nSTDOUT:${cmd_output}\nSTDERR: ${cmd_error}")
         endif()
-        
+
         message("${cmd_output}")
 
         execute_process(COMMAND ${CMAKE_COMMAND} --build .

diff --git a/cpp/perspective/CMakeLists.txt b/cpp/perspective/CMakeLists.txt
@@ -424,6 +424,22 @@ psp_build_dep("date" "${PSP_CMAKE_MODULE_PATH}/date.txt.in")
 psp_build_dep("hopscotch" "${PSP_CMAKE_MODULE_PATH}/hopscotch.txt.in")
 psp_build_dep("ordered-map" "${PSP_CMAKE_MODULE_PATH}/ordered-map.txt.in")
 
+# ExternalProject_Add was not working for zstd, SOURCE_SUBDIR was not being honored.
+include(FetchContent)
+
+set(ZSTD_BUILD_STATIC ON)
+set(ZSTD_BUILD_SHARED OFF)
+set(ZSTD_BUILD_COMPRESSION OFF)
+
+FetchContent_Declare(
+    zstd
+    URL "https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz"
+    DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+    SOURCE_SUBDIR build/cmake
+)
+
+FetchContent_MakeAvailable(zstd)
+
 # Build minimal arrow for both Emscripten and Python
 psp_build_message("${Cyan}Building minimal Apache Arrow${ColorReset}")
 
@@ -733,4 +749,4 @@ endif()
 
 if(NOT WIN32)
     include_directories(SYSTEM ${Boost_INCLUDE_DIRS})
-endif()
+endif()