From 0f5e30845848cdb1f714a0376c59837032686762 Mon Sep 17 00:00:00 2001 From: acezen Date: Thu, 18 May 2023 06:52:48 +0000 Subject: [PATCH 1/3] Fix the arrow acero not found error when building with arrow 12.0.0 or later Signed-off-by: acezen Committed-by: acezen from Dev container --- cpp/CMakeLists.txt | 2 ++ cpp/cmake/apache-arrow.cmake | 27 +++++++++++++++++++-------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 76394df0e..316ef32c0 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -191,10 +191,12 @@ macro(build_gar) if(APPLE) target_link_libraries(gar PRIVATE -Wl,-force_load gar_arrow_static "${GAR_PARQUET_STATIC_LIB}" + "${GAR_ACERO_STATIC_LIB}" "${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB}") else() target_link_libraries(gar PRIVATE -Wl,--exclude-libs,ALL -Wl,--whole-archive gar_arrow_static "${GAR_PARQUET_STATIC_LIB}" + "${GAR_ARROW_ACERO_STATIC_LIB}" "${GAR_ARROW_BUNDLED_DEPS_STATIC_LIB}" -Wl,--no-whole-archive) endif() diff --git a/cpp/cmake/apache-arrow.cmake b/cpp/cmake/apache-arrow.cmake index 377122d70..950f026b6 100644 --- a/cpp/cmake/apache-arrow.cmake +++ b/cpp/cmake/apache-arrow.cmake @@ -33,14 +33,6 @@ function(build_arrow) message(SEND_ERROR "Error: unrecognized arguments: ${ARG_UNPARSED_ARGUMENTS}") endif () - find_package(Threads) - find_package(Arrow QUIET) - set(ARROW_VERSION_TO_BUILD "10.0.1" CACHE INTERNAL "arrow version") - if (Arrow_FOUND) # arrow is installed, build the same version as the installed one - message(STATUS "Found Arrow installed, align to version: ${Arrow_VERSION}") - set(ARROW_VERSION_TO_BUILD "${Arrow_VERSION}" CACHE INTERNAL "arrow version") - endif () - # If Arrow needs to be built, the default location will be within the build tree. set(GAR_ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix") @@ -78,6 +70,7 @@ function(build_arrow) "-DBoost_SOURCE=BUNDLED" "-DARROW_ORC=ON" "-DARROW_COMPUTE=ON" + "-DARROW_ACERO=ON" "-DARROW_DATASET=ON" "-DARROW_JEMALLOC=OFF" "-DARROW_WITH_SNAPPY=OFF" @@ -91,6 +84,14 @@ function(build_arrow) set(GAR_ARROW_INCLUDE_DIR "${GAR_ARROW_PREFIX}/include" CACHE INTERNAL "arrow include directory") set(GAR_ARROW_BUILD_BYPRODUCTS "${GAR_ARROW_STATIC_LIB}" "${GAR_PARQUET_STATIC_LIB}") + + find_package(Threads) + find_package(Arrow QUIET) + set(ARROW_VERSION_TO_BUILD "10.0.1" CACHE INTERNAL "arrow version") + if (Arrow_FOUND) # arrow is installed, build the same version as the installed one + message(STATUS "Found Arrow installed, align to version: ${Arrow_VERSION}") + set(ARROW_VERSION_TO_BUILD "${Arrow_VERSION}" CACHE INTERNAL "arrow version") + endif () set(GAR_ARROW_SOURCE_FILE "https://www.apache.org/dyn/closer.lua?action=download&filename=arrow/arrow-${ARROW_VERSION_TO_BUILD}/apache-arrow-${ARROW_VERSION_TO_BUILD}.tar.gz") include(ExternalProject) @@ -113,6 +114,16 @@ function(build_arrow) set_target_properties(${GAR_PARQUET_LIBRARY_TARGET} PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${GAR_ARROW_INCLUDE_DIR} IMPORTED_LOCATION ${GAR_PARQUET_STATIC_LIB}) + if (ARROW_VERSION_TO_BUILD GREATER_EQUAL "12.0.0") + set(GAR_ARROW_ACERO_STATIC_LIB_FILENAME + "${CMAKE_STATIC_LIBRARY_PREFIX}arrow_acero${CMAKE_STATIC_LIBRARY_SUFFIX}") + set(GAR_ARROW_ACERO_STATIC_LIB "${GAR_ARROW_STATIC_LIBRARY_DIR}/${GAR_ARROW_ACERO_STATIC_LIB_FILENAME}" CACHE INTERNAL "acero lib") + set(GAR_ARROW_ACERO_LIBRARY_TARGET gar_acero_static) + add_library(${GAR_ARROW_ACERO_LIBRARY_TARGET} STATIC IMPORTED) + set_target_properties(${GAR_ARROW_ACERO_LIBRARY_TARGET} + PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${GAR_ARROW_INCLUDE_DIR} + IMPORTED_LOCATION ${GAR_ARROW_ACERO_STATIC_LIB}) + endif() add_dependencies(${GAR_ARROW_LIBRARY_TARGET} arrow_ep) endfunction() From 2c7ae4fa3b46a07a070ee7ec8927674dcf378a95 Mon Sep 17 00:00:00 2001 From: acezen Date: Thu, 18 May 2023 07:01:44 +0000 Subject: [PATCH 2/3] Add test to ci Signed-off-by: acezen Committed-by: acezen from Dev container --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d3045e66..0ea9d7dd2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,7 +46,13 @@ jobs: - name: Install dependencies run: | + + # install the latest arrow deb to test arrow + wget -c https://apache.jfrog.io/artifactory/arrow/"$(lsb_release --id --short | tr 'A-Z' 'a-z')"/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb \ + -P /tmp/ + apt-get install -y -V /tmp/apache-arrow-apt-source-latest-"$(lsb_release --codename --short)".deb sudo apt-get update -y + apt-get install -y libarrow-dev sudo apt-get install -y libboost-graph-dev ccache libcurl4-openssl-dev - name: CMake From 6c13a97e8bcb9a3b8caf6d69a66c20f78c6315ad Mon Sep 17 00:00:00 2001 From: acezen Date: Thu, 18 May 2023 07:36:27 +0000 Subject: [PATCH 3/3] Fix Signed-off-by: acezen Committed-by: acezen from Dev container --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ea9d7dd2..6f582db8b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,9 +50,9 @@ jobs: # install the latest arrow deb to test arrow wget -c https://apache.jfrog.io/artifactory/arrow/"$(lsb_release --id --short | tr 'A-Z' 'a-z')"/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb \ -P /tmp/ - apt-get install -y -V /tmp/apache-arrow-apt-source-latest-"$(lsb_release --codename --short)".deb + sudo apt-get install -y -V /tmp/apache-arrow-apt-source-latest-"$(lsb_release --codename --short)".deb sudo apt-get update -y - apt-get install -y libarrow-dev + sudo apt-get install -y libarrow-dev sudo apt-get install -y libboost-graph-dev ccache libcurl4-openssl-dev - name: CMake