From c084c96539072f13daea8bf5e08f06c306aec3d4 Mon Sep 17 00:00:00 2001 From: Davis Silverman Date: Fri, 23 Aug 2024 15:23:57 -0400 Subject: [PATCH] Zstd decompression for arrows Signed-off-by: Davis Silverman --- cmake/arrow/CMakeLists.txt | 11 +++++++---- cmake/modules/FindInstallDependency.cmake | 4 ++-- cpp/perspective/CMakeLists.txt | 18 +++++++++++++++++- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/cmake/arrow/CMakeLists.txt b/cmake/arrow/CMakeLists.txt index 6830e0fbc7..b4ebab4534 100644 --- a/cmake/arrow/CMakeLists.txt +++ b/cmake/arrow/CMakeLists.txt @@ -90,7 +90,7 @@ set(ARROW_SRCS ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/byte_size.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/cancel.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/compression.cc - # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/compression_zstd.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/compression_zstd.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/compression_lz4.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/counting_semaphore.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/debug.cc @@ -154,11 +154,11 @@ set(ARROW_SRCS ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/exec.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/expression.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/function.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/function_internal.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/function_internal.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernel.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/ordering.cc - + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/registry.cc # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/aggregate_basic.cc # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/aggregate_mode.cc @@ -278,9 +278,11 @@ add_library(arrow STATIC ${ARROW_SRCS}) target_compile_definitions(arrow PUBLIC ARROW_NO_DEPRECATED_API) target_compile_definitions(arrow PUBLIC ARROW_STATIC) -# target_compile_definitions(arrow PUBLIC ARROW_WITH_ZSTD=ON) +target_compile_definitions(arrow PUBLIC ARROW_WITH_ZSTD=ON) target_compile_definitions(arrow PUBLIC ARROW_WITH_LZ4) +target_include_directories(arrow SYSTEM PRIVATE ${zstd_SOURCE_DIR}/lib) + # will need built boost filesystem and system .lib to work, even though # perspective itself does not use those dependencies target_link_libraries(arrow @@ -288,6 +290,7 @@ target_link_libraries(arrow ${Boost_FILESYSTEM_LIBRARY} ${Boost_SYSTEM_LIBRARY} lz4_static + libzstd_static ${ARROW_TEST_LINK_TOOLCHAIN}) # find_package(Flatbuffers) diff --git a/cmake/modules/FindInstallDependency.cmake b/cmake/modules/FindInstallDependency.cmake index 83bc3794a2..df883bf020 100644 --- a/cmake/modules/FindInstallDependency.cmake +++ b/cmake/modules/FindInstallDependency.cmake @@ -12,7 +12,7 @@ function(psp_build_dep name cmake_file) else() configure_file(${cmake_file} ${CMAKE_BINARY_DIR}/${name}-download/CMakeLists.txt) set(_cwd ${CMAKE_BINARY_DIR}/${name}-download) - + message(STATUS "Configuring ${name} in ${_cwd}") execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" . @@ -24,7 +24,7 @@ function(psp_build_dep name cmake_file) if(result) message(FATAL_ERROR "CMake step for ${name} failed:\nSTDOUT:${cmd_output}\nSTDERR: ${cmd_error}") endif() - + message("${cmd_output}") execute_process(COMMAND ${CMAKE_COMMAND} --build . diff --git a/cpp/perspective/CMakeLists.txt b/cpp/perspective/CMakeLists.txt index eb964a2ddc..4abf7ab86b 100644 --- a/cpp/perspective/CMakeLists.txt +++ b/cpp/perspective/CMakeLists.txt @@ -424,6 +424,22 @@ psp_build_dep("date" "${PSP_CMAKE_MODULE_PATH}/date.txt.in") psp_build_dep("hopscotch" "${PSP_CMAKE_MODULE_PATH}/hopscotch.txt.in") psp_build_dep("ordered-map" "${PSP_CMAKE_MODULE_PATH}/ordered-map.txt.in") +# ExternalProject_Add was not working for zstd, SOURCE_SUBDIR was not being honored. +include(FetchContent) + +set(ZSTD_BUILD_STATIC ON) +set(ZSTD_BUILD_SHARED OFF) +set(ZSTD_BUILD_COMPRESSION OFF) + +FetchContent_Declare( + zstd + URL "https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz" + DOWNLOAD_EXTRACT_TIMESTAMP TRUE + SOURCE_SUBDIR build/cmake +) + +FetchContent_MakeAvailable(zstd) + # Build minimal arrow for both Emscripten and Python psp_build_message("${Cyan}Building minimal Apache Arrow${ColorReset}") @@ -733,4 +749,4 @@ endif() if(NOT WIN32) include_directories(SYSTEM ${Boost_INCLUDE_DIRS}) -endif() \ No newline at end of file +endif()