From 90575addccec05da6e884d90023aaa3619498819 Mon Sep 17 00:00:00 2001 From: Davis Silverman Date: Fri, 23 Aug 2024 15:23:57 -0400 Subject: [PATCH] Zstd decompression for arrows Signed-off-by: Davis Silverman --- cmake/arrow/CMakeLists.txt | 10 ++++++---- cmake/modules/FindInstallDependency.cmake | 4 ++-- cpp/perspective/CMakeLists.txt | 20 ++++++++++++++++++-- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/cmake/arrow/CMakeLists.txt b/cmake/arrow/CMakeLists.txt index 6830e0fbc7..96737f1233 100644 --- a/cmake/arrow/CMakeLists.txt +++ b/cmake/arrow/CMakeLists.txt @@ -90,7 +90,7 @@ set(ARROW_SRCS ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/byte_size.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/cancel.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/compression.cc - # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/compression_zstd.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/compression_zstd.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/compression_lz4.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/counting_semaphore.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/debug.cc @@ -154,11 +154,11 @@ set(ARROW_SRCS ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/exec.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/expression.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/function.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/function_internal.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/function_internal.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernel.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/ordering.cc - + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/registry.cc # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/aggregate_basic.cc # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/aggregate_mode.cc @@ -278,9 +278,11 @@ add_library(arrow STATIC ${ARROW_SRCS}) target_compile_definitions(arrow PUBLIC ARROW_NO_DEPRECATED_API) target_compile_definitions(arrow PUBLIC ARROW_STATIC) -# target_compile_definitions(arrow PUBLIC ARROW_WITH_ZSTD=ON) +target_compile_definitions(arrow PUBLIC ARROW_WITH_ZSTD=ON) target_compile_definitions(arrow PUBLIC ARROW_WITH_LZ4) +target_include_directories(arrow SYSTEM PRIVATE ${zstd_SOURCE_DIR}/lib) + # will need built boost filesystem and system .lib to work, even though # perspective itself does not use those dependencies target_link_libraries(arrow diff --git a/cmake/modules/FindInstallDependency.cmake b/cmake/modules/FindInstallDependency.cmake index 83bc3794a2..df883bf020 100644 --- a/cmake/modules/FindInstallDependency.cmake +++ b/cmake/modules/FindInstallDependency.cmake @@ -12,7 +12,7 @@ function(psp_build_dep name cmake_file) else() configure_file(${cmake_file} ${CMAKE_BINARY_DIR}/${name}-download/CMakeLists.txt) set(_cwd ${CMAKE_BINARY_DIR}/${name}-download) - + message(STATUS "Configuring ${name} in ${_cwd}") execute_process(COMMAND ${CMAKE_COMMAND} -G "${CMAKE_GENERATOR}" . @@ -24,7 +24,7 @@ function(psp_build_dep name cmake_file) if(result) message(FATAL_ERROR "CMake step for ${name} failed:\nSTDOUT:${cmd_output}\nSTDERR: ${cmd_error}") endif() - + message("${cmd_output}") execute_process(COMMAND ${CMAKE_COMMAND} --build . diff --git a/cpp/perspective/CMakeLists.txt b/cpp/perspective/CMakeLists.txt index eb964a2ddc..7baa8c2e87 100644 --- a/cpp/perspective/CMakeLists.txt +++ b/cpp/perspective/CMakeLists.txt @@ -424,6 +424,22 @@ psp_build_dep("date" "${PSP_CMAKE_MODULE_PATH}/date.txt.in") psp_build_dep("hopscotch" "${PSP_CMAKE_MODULE_PATH}/hopscotch.txt.in") psp_build_dep("ordered-map" "${PSP_CMAKE_MODULE_PATH}/ordered-map.txt.in") +# ExternalProject_Add was not working for zstd, SOURCE_SUBDIR was not being honored. +include(FetchContent) + +set(ZSTD_BUILD_STATIC ON) +set(ZSTD_BUILD_SHARED OFF) +set(ZSTD_BUILD_COMPRESSION OFF) + +FetchContent_Declare( + zstd + URL "https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz" + DOWNLOAD_EXTRACT_TIMESTAMP TRUE + SOURCE_SUBDIR build/cmake +) + +FetchContent_MakeAvailable(zstd) + # Build minimal arrow for both Emscripten and Python psp_build_message("${Cyan}Building minimal Apache Arrow${ColorReset}") @@ -624,7 +640,7 @@ if(PSP_WASM_BUILD AND NOT PSP_PYTHON_BUILD) add_library(psp ${WASM_SOURCE_FILES}) target_compile_definitions(psp PRIVATE PSP_ENABLE_WASM=1) set_target_properties(psp PROPERTIES COMPILE_FLAGS "") - target_link_libraries(psp PRIVATE arrow re2 protos) + target_link_libraries(psp PRIVATE arrow re2 protos libzstd_static) add_executable(perspective_esm src/cpp/binding_api.cpp) target_link_libraries(perspective_esm psp protos) @@ -733,4 +749,4 @@ endif() if(NOT WIN32) include_directories(SYSTEM ${Boost_INCLUDE_DIRS}) -endif() \ No newline at end of file +endif()