From 8215abc420121f79e06996898a4b226474f7ca99 Mon Sep 17 00:00:00 2001 From: Leif Walsh Date: Sat, 8 Oct 2016 18:12:44 -0400 Subject: [PATCH 1/4] ARROW-189: Build 3rd party with ExternalProject. When third party env vars *_HOME are not present, use cmake's ExternalProject to fetch and build them. When those vars are present, we just use them. --- cpp/CMakeLists.txt | 111 ++++++++++++++++++++++++++----- cpp/src/arrow/ipc/CMakeLists.txt | 27 ++++++-- 2 files changed, 113 insertions(+), 25 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index f70c8ab4bcc..8c7f5d657ba 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -21,9 +21,13 @@ project(arrow) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules") include(CMakeParseArguments) +include(ExternalProject) set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support") -set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty") + +set(GTEST_VERSION "1.7.0") +set(GBENCHMARK_VERSION "1.0.0") +set(FLATBUFFERS_VERSION "1.3.0") find_package(ClangTools) if ("$ENV{CMAKE_EXPORT_COMPILE_COMMANDS}" STREQUAL "1" OR CLANG_TIDY_FOUND) @@ -422,16 +426,6 @@ function(ADD_THIRDPARTY_LIB LIB_NAME) endif() endfunction() -## GTest -if ("$ENV{GTEST_HOME}" STREQUAL "") - set(GTest_HOME ${THIRDPARTY_DIR}/googletest-release-1.7.0) -endif() - -## Google Benchmark -if ("$ENV{GBENCHMARK_HOME}" STREQUAL "") - set(GBENCHMARK_HOME ${THIRDPARTY_DIR}/installed) -endif() - # ---------------------------------------------------------------------- # Add Boost dependencies (code adapted from Apache Kudu (incubating)) @@ -476,18 +470,78 @@ include_directories(SYSTEM ${Boost_INCLUDE_DIR}) if(ARROW_BUILD_TESTS) add_custom_target(unittest ctest -L unittest) - find_package(GTest REQUIRED) + + if("$ENV{GTEST_HOME}" STREQUAL "") + if(APPLE) + set(GTEST_CMAKE_ARGS "-DCMAKE_CXX_FLAGS=\"-fPIC -std=c++11 -stdlib=libc++ -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-value -Wno-ignored-attributes\"") + else() + set(GTEST_CMAKE_ARGS "-DCMAKE_CXX_FLAGS=\"-fPIC\"") + endif() + + ExternalProject_Add(googletest_ep + URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz" + CMAKE_ARGS ${GTEST_CMAKE_ARGS} + # googletest doesn't define install rules, so just build in the + # source dir and don't try to install. See its README for + # details. + BUILD_IN_SOURCE 1 + INSTALL_COMMAND "/bin/true") + + set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep") + set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include") + set(GTEST_STATIC_LIB "${GTEST_PREFIX}/libgtest.a") + add_library(gtest STATIC IMPORTED) + set_target_properties(gtest PROPERTIES + IMPORTED_LOCATION ${GTEST_STATIC_LIB}) + add_dependencies(gtest googletest_ep) + set(GTEST_VENDORED 1) + else() + find_package(GTest REQUIRED) + ADD_THIRDPARTY_LIB(gtest + STATIC_LIB ${GTEST_STATIC_LIB}) + set(GTEST_VENDORED 0) + endif() + + message(STATUS "GTest include dir: ${GTEST_INCLUDE_DIR}") + message(STATUS "GTest static library: ${GTEST_STATIC_LIB}") include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) - ADD_THIRDPARTY_LIB(gtest - STATIC_LIB ${GTEST_STATIC_LIB}) endif() if(ARROW_BUILD_BENCHMARKS) add_custom_target(runbenchmark ctest -L benchmark) - find_package(GBenchmark REQUIRED) + + if("$ENV{GBENCHMARK_HOME}" STREQUAL "") + if(APPLE) + set(GBENCHMARK_CMAKE_CXX_FLAGS "-std=c++11 -stdlib=libc++") + else() + set(GBENCHMARK_CMAKE_CXX_FLAGS "--std=c++11") + endif() + + set(GBENCHMARK_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/gbenchmark_ep/src/gbenchmark_ep-install") + ExternalProject_Add(gbenchmark_ep + URL "https://github.com/google/benchmark/archive/v${GBENCHMARK_VERSION}.tar.gz" + CMAKE_ARGS + "-DCMAKE_BUILD_TYPE=Release" + "-DCMAKE_INSTALL_PREFIX:PATH=${GBENCHMARK_PREFIX}" + "-DCMAKE_CXX_FLAGS=-fPIC ${GBENCHMARK_CMAKE_CXX_FLAGS}") + + set(GBENCHMARK_INCLUDE_DIR "${GBENCHMARK_PREFIX}/include") + set(GBENCHMARK_STATIC_LIB "${GBENCHMARK_PREFIX}/lib/libbenchmark.a") + add_library(benchmark STATIC IMPORTED) + set_target_properties(benchmark PROPERTIES + IMPORTED_LOCATION ${GBENCHMARK_STATIC_LIB}) + add_dependencies(benchmark gbenchmark_ep) + set(GBENCHMARK_VENDORED 1) + else() + find_package(GBenchmark REQUIRED) + ADD_THIRDPARTY_LIB(benchmark + STATIC_LIB ${GBENCHMARK_STATIC_LIB}) + set(GBENCHMARK_VENDORED 0) + endif() + + message(STATUS "GBenchmark include dir: ${GBENCHMARK_INCLUDE_DIR}") + message(STATUS "GBenchmark static library: ${GBENCHMARK_STATIC_LIB}") include_directories(SYSTEM ${GBENCHMARK_INCLUDE_DIR}) - ADD_THIRDPARTY_LIB(benchmark - STATIC_LIB ${GBENCHMARK_STATIC_LIB}) endif() ## Google PerfTools @@ -705,7 +759,24 @@ add_subdirectory(src/arrow/types) ## Flatbuffers if(ARROW_IPC) - find_package(Flatbuffers REQUIRED) + if("$ENV{FLATBUFFERS_HOME}" STREQUAL "") + set(FLATBUFFERS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/flatbuffers_ep-prefix/src/flatbuffers_ep-install") + ExternalProject_Add(flatbuffers_ep + URL "https://github.com/google/flatbuffers/archive/v${FLATBUFFERS_VERSION}.tar.gz" + CMAKE_ARGS + "-DCMAKE_CXX_FLAGS=-fPIC" + "-DCMAKE_INSTALL_PREFIX:PATH=${FLATBUFFERS_PREFIX}" + "-DFLATBUFFERS_BUILD_TESTS=OFF") + + set(FLATBUFFERS_INCLUDE_DIR "${FLATBUFFERS_PREFIX}/include") + set(FLATBUFFERS_STATIC_LIB "${FLATBUFFERS_PREFIX}/libflatbuffers.a") + set(FLATBUFFERS_COMPILER "${FLATBUFFERS_PREFIX}/bin/flatc") + set(FLATBUFFERS_VENDORED 1) + else() + find_package(Flatbuffers REQUIRED) + set(FLATBUFFERS_VENDORED 0) + endif() + message(STATUS "Flatbuffers include dir: ${FLATBUFFERS_INCLUDE_DIR}") message(STATUS "Flatbuffers static library: ${FLATBUFFERS_STATIC_LIB}") message(STATUS "Flatbuffers compiler: ${FLATBUFFERS_COMPILER}") @@ -714,5 +785,9 @@ if(ARROW_IPC) set_target_properties(flatbuffers PROPERTIES IMPORTED_LOCATION ${FLATBUFFERS_STATIC_LIB}) + if(FLATBUFFERS_VENDORED) + add_dependencies(flatbuffers flatbuffers_ep) + endif() + add_subdirectory(src/arrow/ipc) endif() diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt index bde8c5bf738..91209b6f5eb 100644 --- a/cpp/src/arrow/ipc/CMakeLists.txt +++ b/cpp/src/arrow/ipc/CMakeLists.txt @@ -42,6 +42,9 @@ set(ARROW_IPC_SRCS add_library(arrow_ipc SHARED ${ARROW_IPC_SRCS} ) +if(FLATBUFFERS_VENDORED) + add_dependencies(arrow_ipc flatbuffers_ep) +endif() target_link_libraries(arrow_ipc LINK_PUBLIC ${ARROW_IPC_LINK_LIBS} LINK_PRIVATE ${ARROW_IPC_PRIVATE_LINK_LIBS}) @@ -84,13 +87,23 @@ foreach(FIL ${FBS_SRC}) list(APPEND ABS_FBS_SRC ${ABS_FIL}) endforeach() -add_custom_command( - OUTPUT ${FBS_OUTPUT_FILES} - COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${ABS_FBS_SRC} - DEPENDS ${ABS_FBS_SRC} - COMMENT "Running flatc compiler on ${ABS_FBS_SRC}" - VERBATIM -) +if(FLATBUFFERS_VENDORED) + add_custom_command( + OUTPUT ${FBS_OUTPUT_FILES} + COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${ABS_FBS_SRC} + DEPENDS ${ABS_FBS_SRC} flatbuffers_ep + COMMENT "Running flatc compiler on ${ABS_FBS_SRC}" + VERBATIM + ) +else() + add_custom_command( + OUTPUT ${FBS_OUTPUT_FILES} + COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${ABS_FBS_SRC} + DEPENDS ${ABS_FBS_SRC} + COMMENT "Running flatc compiler on ${ABS_FBS_SRC}" + VERBATIM + ) +endif() add_custom_target(metadata_fbs DEPENDS ${FBS_OUTPUT_FILES}) add_dependencies(arrow_objlib metadata_fbs) From 8630428ed17f4bdbcd2a300516afb3f1047a0072 Mon Sep 17 00:00:00 2001 From: Leif Walsh Date: Tue, 11 Oct 2016 05:59:53 -0400 Subject: [PATCH 2/4] ARROW-189: Addressed CR comments. - Removed thirdparty build scripts and references to them. - Fixed ARROW_HDFS=1 builds (THIRDPARTY_DIR still needs to be set). - Simplified some cmake rules. --- ci/travis_before_script_cpp.sh | 8 -- ci/travis_script_python.sh | 5 -- cpp/CMakeLists.txt | 30 ++++---- cpp/README.md | 18 +---- cpp/doc/Parquet.md | 1 - cpp/setup_build_env.sh | 21 ------ cpp/src/arrow/ipc/CMakeLists.txt | 23 +++--- cpp/thirdparty/build_thirdparty.sh | 104 -------------------------- cpp/thirdparty/download_thirdparty.sh | 44 ----------- cpp/thirdparty/set_thirdparty_env.sh | 24 ------ cpp/thirdparty/versions.sh | 23 ------ 11 files changed, 25 insertions(+), 276 deletions(-) delete mode 100755 cpp/setup_build_env.sh delete mode 100755 cpp/thirdparty/build_thirdparty.sh delete mode 100755 cpp/thirdparty/download_thirdparty.sh delete mode 100755 cpp/thirdparty/set_thirdparty_env.sh delete mode 100755 cpp/thirdparty/versions.sh diff --git a/ci/travis_before_script_cpp.sh b/ci/travis_before_script_cpp.sh index acd820bbed2..2d4224b3333 100755 --- a/ci/travis_before_script_cpp.sh +++ b/ci/travis_before_script_cpp.sh @@ -26,14 +26,6 @@ pushd $CPP_BUILD_DIR CPP_DIR=$TRAVIS_BUILD_DIR/cpp -# Build an isolated thirdparty -cp -r $CPP_DIR/thirdparty . -cp $CPP_DIR/setup_build_env.sh . - -source setup_build_env.sh - -echo $GTEST_HOME - : ${ARROW_CPP_INSTALL=$TRAVIS_BUILD_DIR/cpp-install} CMAKE_COMMON_FLAGS="\ diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh index a75ff0778bc..97f0563240c 100755 --- a/ci/travis_script_python.sh +++ b/ci/travis_script_python.sh @@ -21,11 +21,6 @@ export MINICONDA=$HOME/miniconda export PATH="$MINICONDA/bin:$PATH" export PARQUET_HOME=$MINICONDA -# Share environment with C++ -pushd $CPP_BUILD_DIR -source setup_build_env.sh -popd - pushd $PYTHON_DIR python_version_tests() { diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 8c7f5d657ba..df4c8bd17d1 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -24,6 +24,7 @@ include(CMakeParseArguments) include(ExternalProject) set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support") +set(THIRDPARTY_DIR "${CMAKE_SOURCE_DIR}/thirdparty") set(GTEST_VERSION "1.7.0") set(GBENCHMARK_VERSION "1.0.0") @@ -490,21 +491,21 @@ if(ARROW_BUILD_TESTS) set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep") set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include") set(GTEST_STATIC_LIB "${GTEST_PREFIX}/libgtest.a") - add_library(gtest STATIC IMPORTED) - set_target_properties(gtest PROPERTIES - IMPORTED_LOCATION ${GTEST_STATIC_LIB}) - add_dependencies(gtest googletest_ep) set(GTEST_VENDORED 1) else() find_package(GTest REQUIRED) - ADD_THIRDPARTY_LIB(gtest - STATIC_LIB ${GTEST_STATIC_LIB}) set(GTEST_VENDORED 0) endif() message(STATUS "GTest include dir: ${GTEST_INCLUDE_DIR}") message(STATUS "GTest static library: ${GTEST_STATIC_LIB}") include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) + ADD_THIRDPARTY_LIB(gtest + STATIC_LIB ${GTEST_STATIC_LIB}) + + if(GTEST_VENDORED) + add_dependencies(gtest googletest_ep) + endif() endif() if(ARROW_BUILD_BENCHMARKS) @@ -527,21 +528,21 @@ if(ARROW_BUILD_BENCHMARKS) set(GBENCHMARK_INCLUDE_DIR "${GBENCHMARK_PREFIX}/include") set(GBENCHMARK_STATIC_LIB "${GBENCHMARK_PREFIX}/lib/libbenchmark.a") - add_library(benchmark STATIC IMPORTED) - set_target_properties(benchmark PROPERTIES - IMPORTED_LOCATION ${GBENCHMARK_STATIC_LIB}) - add_dependencies(benchmark gbenchmark_ep) set(GBENCHMARK_VENDORED 1) else() find_package(GBenchmark REQUIRED) - ADD_THIRDPARTY_LIB(benchmark - STATIC_LIB ${GBENCHMARK_STATIC_LIB}) set(GBENCHMARK_VENDORED 0) endif() message(STATUS "GBenchmark include dir: ${GBENCHMARK_INCLUDE_DIR}") message(STATUS "GBenchmark static library: ${GBENCHMARK_STATIC_LIB}") include_directories(SYSTEM ${GBENCHMARK_INCLUDE_DIR}) + ADD_THIRDPARTY_LIB(benchmark + STATIC_LIB ${GBENCHMARK_STATIC_LIB}) + + if(GBENCHMARK_VENDORED) + add_dependencies(benchmark gbenchmark_ep) + endif() endif() ## Google PerfTools @@ -781,9 +782,8 @@ if(ARROW_IPC) message(STATUS "Flatbuffers static library: ${FLATBUFFERS_STATIC_LIB}") message(STATUS "Flatbuffers compiler: ${FLATBUFFERS_COMPILER}") include_directories(SYSTEM ${FLATBUFFERS_INCLUDE_DIR}) - add_library(flatbuffers STATIC IMPORTED) - set_target_properties(flatbuffers PROPERTIES - IMPORTED_LOCATION ${FLATBUFFERS_STATIC_LIB}) + ADD_THIRDPARTY_LIB(flatbuffers + STATIC_LIB ${FLATBUFFERS_STATIC_LIB}) if(FLATBUFFERS_VENDORED) add_dependencies(flatbuffers flatbuffers_ep) diff --git a/cpp/README.md b/cpp/README.md index a1c3ef28447..190e6f85b42 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -22,23 +22,6 @@ out-of-source builds with the latter one being preferred. Arrow requires a C++11-enabled compiler. On Linux, gcc 4.8 and higher should be sufficient. -To build the thirdparty build dependencies, run: - -``` -./thirdparty/download_thirdparty.sh -./thirdparty/build_thirdparty.sh -source ./thirdparty/set_thirdparty_env.sh -``` - -You can also run from the root of the C++ tree - -``` -source setup_build_env.sh -``` - -Arrow is configured to use the `thirdparty` directory by default for its build -dependencies. To set up a custom toolchain see below. - Simple debug build: mkdir debug @@ -76,6 +59,7 @@ variables * Googletest: `GTEST_HOME` (only required to build the unit tests) * Google Benchmark: `GBENCHMARK_HOME` (only required if building benchmarks) * Flatbuffers: `FLATBUFFERS_HOME` (only required for the IPC extensions) +* Hadoop: `HADOOP_HOME` (only required for the HDFS I/O extensions) ## Continuous Integration diff --git a/cpp/doc/Parquet.md b/cpp/doc/Parquet.md index 34b83e78d0a..4985dd3b0bc 100644 --- a/cpp/doc/Parquet.md +++ b/cpp/doc/Parquet.md @@ -24,7 +24,6 @@ export ARROW_HOME=$HOME/local git clone https://github.com/apache/parquet-cpp.git cd parquet-cpp -source setup_build_env.sh cmake -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME -DPARQUET_ARROW=on make -j4 make install diff --git a/cpp/setup_build_env.sh b/cpp/setup_build_env.sh deleted file mode 100755 index 546216753b3..00000000000 --- a/cpp/setup_build_env.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. See accompanying LICENSE file. - -SOURCE_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) - -./thirdparty/download_thirdparty.sh || { echo "download_thirdparty.sh failed" ; return; } -./thirdparty/build_thirdparty.sh || { echo "build_thirdparty.sh failed" ; return; } -source ./thirdparty/set_thirdparty_env.sh || { echo "source set_thirdparty_env.sh failed" ; return; } - -echo "Build env initialized" diff --git a/cpp/src/arrow/ipc/CMakeLists.txt b/cpp/src/arrow/ipc/CMakeLists.txt index 91209b6f5eb..0557addc5ed 100644 --- a/cpp/src/arrow/ipc/CMakeLists.txt +++ b/cpp/src/arrow/ipc/CMakeLists.txt @@ -88,22 +88,17 @@ foreach(FIL ${FBS_SRC}) endforeach() if(FLATBUFFERS_VENDORED) - add_custom_command( - OUTPUT ${FBS_OUTPUT_FILES} - COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${ABS_FBS_SRC} - DEPENDS ${ABS_FBS_SRC} flatbuffers_ep - COMMENT "Running flatc compiler on ${ABS_FBS_SRC}" - VERBATIM - ) + set(FBS_DEPENDS ${ABS_FBS_SRC} flatbuffers_ep) else() - add_custom_command( - OUTPUT ${FBS_OUTPUT_FILES} - COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${ABS_FBS_SRC} - DEPENDS ${ABS_FBS_SRC} - COMMENT "Running flatc compiler on ${ABS_FBS_SRC}" - VERBATIM - ) + set(FBS_DEPENDS ${ABS_FBS_SRC}) endif() +add_custom_command( + OUTPUT ${FBS_OUTPUT_FILES} + COMMAND ${FLATBUFFERS_COMPILER} -c -o ${OUTPUT_DIR} ${ABS_FBS_SRC} + DEPENDS ${FBS_DEPENDS} + COMMENT "Running flatc compiler on ${ABS_FBS_SRC}" + VERBATIM +) add_custom_target(metadata_fbs DEPENDS ${FBS_OUTPUT_FILES}) add_dependencies(arrow_objlib metadata_fbs) diff --git a/cpp/thirdparty/build_thirdparty.sh b/cpp/thirdparty/build_thirdparty.sh deleted file mode 100755 index 5011e29c01a..00000000000 --- a/cpp/thirdparty/build_thirdparty.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. See accompanying LICENSE file. - -set -x -set -e -TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) - -source $TP_DIR/versions.sh -PREFIX=$TP_DIR/installed - -################################################################################ - -if [ "$#" = "0" ]; then - F_ALL=1 -else - # Allow passing specific libs to build on the command line - for arg in "$*"; do - case $arg in - "gtest") F_GTEST=1 ;; - "gbenchmark") F_GBENCHMARK=1 ;; - "flatbuffers") F_FLATBUFFERS=1 ;; - *) echo "Unknown module: $arg"; exit 1 ;; - esac - done -fi - -################################################################################ - -# Determine how many parallel jobs to use for make based on the number of cores -if [[ "$OSTYPE" =~ ^linux ]]; then - PARALLEL=$(grep -c processor /proc/cpuinfo) -elif [[ "$OSTYPE" == "darwin"* ]]; then - PARALLEL=$(sysctl -n hw.ncpu) -else - echo Unsupported platform $OSTYPE - exit 1 -fi - -mkdir -p "$PREFIX/include" -mkdir -p "$PREFIX/lib" - -# On some systems, autotools installs libraries to lib64 rather than lib. Fix -# this by setting up lib64 as a symlink to lib. We have to do this step first -# to handle cases where one third-party library depends on another. -ln -sf lib "$PREFIX/lib64" - -# use the compiled tools -export PATH=$PREFIX/bin:$PATH - -type cmake >/dev/null 2>&1 || { echo >&2 "cmake not installed. Aborting."; exit 1; } -type make >/dev/null 2>&1 || { echo >&2 "make not installed. Aborting."; exit 1; } - -STANDARD_DARWIN_FLAGS="-std=c++11 -stdlib=libc++" - -# build googletest -GOOGLETEST_ERROR="failed for googletest!" -if [ -n "$F_ALL" -o -n "$F_GTEST" ]; then - cd $TP_DIR/$GTEST_BASEDIR - - if [[ "$OSTYPE" == "darwin"* ]]; then - CXXFLAGS=-fPIC cmake -DCMAKE_CXX_FLAGS="$STANDARD_DARWIN_FLAGS -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-value -Wno-ignored-attributes" || { echo "cmake $GOOGLETEST_ERROR" ; exit 1; } - else - CXXFLAGS=-fPIC cmake . || { echo "cmake $GOOGLETEST_ERROR"; exit 1; } - fi - - make -j$PARALLEL VERBOSE=1 || { echo "Make $GOOGLETEST_ERROR" ; exit 1; } -fi - -# build google benchmark -GBENCHMARK_ERROR="failed for google benchmark" -if [ -n "$F_ALL" -o -n "$F_GBENCHMARK" ]; then - cd $TP_DIR/$GBENCHMARK_BASEDIR - - CMAKE_CXX_FLAGS="--std=c++11" - if [[ "$OSTYPE" == "darwin"* ]]; then - CMAKE_CXX_FLAGS=$STANDARD_DARWIN_FLAGS - fi - cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$PREFIX -DCMAKE_CXX_FLAGS="-fPIC $CMAKE_CXX_FLAGS" . || { echo "cmake $GBENCHMARK_ERROR" ; exit 1; } - - make -j$PARALLEL VERBOSE=1 install || { echo "make $GBENCHMARK_ERROR" ; exit 1; } -fi - -FLATBUFFERS_ERROR="failed for flatbuffers" -if [ -n "$F_ALL" -o -n "$F_FLATBUFFERS" ]; then - cd $TP_DIR/$FLATBUFFERS_BASEDIR - - CXXFLAGS=-fPIC cmake -DCMAKE_INSTALL_PREFIX:PATH=$PREFIX -DFLATBUFFERS_BUILD_TESTS=OFF . || { echo "cmake $FLATBUFFERS_ERROR" ; exit 1; } - make VERBOSE=1 -j$PARALLEL || { echo "make $FLATBUFFERS_ERROR" ; exit 1; } - make install || { echo "install $FLATBUFFERS_ERROR" ; exit 1; } -fi - -echo "---------------------" -echo "Thirdparty dependencies built and installed into $PREFIX successfully" diff --git a/cpp/thirdparty/download_thirdparty.sh b/cpp/thirdparty/download_thirdparty.sh deleted file mode 100755 index b50e7bc06a1..00000000000 --- a/cpp/thirdparty/download_thirdparty.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. See accompanying LICENSE file. - -set -x -set -e - -TP_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) - -source $TP_DIR/versions.sh - -download_extract_and_cleanup() { - type curl >/dev/null 2>&1 || { echo >&2 "curl not installed. Aborting."; exit 1; } - filename=$TP_DIR/$(basename "$1") - curl -#LC - "$1" -o $filename - tar xzf $filename -C $TP_DIR - rm $filename -} - -if [ ! -d ${GTEST_BASEDIR} ]; then - echo "Fetching gtest" - download_extract_and_cleanup $GTEST_URL -fi - -echo ${GBENCHMARK_BASEDIR} -if [ ! -d ${GBENCHMARK_BASEDIR} ]; then - echo "Fetching google benchmark" - download_extract_and_cleanup $GBENCHMARK_URL -fi - -if [ ! -d ${FLATBUFFERS_BASEDIR} ]; then - echo "Fetching flatbuffers" - download_extract_and_cleanup $FLATBUFFERS_URL -fi diff --git a/cpp/thirdparty/set_thirdparty_env.sh b/cpp/thirdparty/set_thirdparty_env.sh deleted file mode 100755 index 135972ee9bd..00000000000 --- a/cpp/thirdparty/set_thirdparty_env.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bash - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. See accompanying LICENSE file. - -SOURCE_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -source $SOURCE_DIR/versions.sh - -if [ -z "$THIRDPARTY_DIR" ]; then - THIRDPARTY_DIR=$SOURCE_DIR -fi - -export GTEST_HOME=$THIRDPARTY_DIR/$GTEST_BASEDIR -export GBENCHMARK_HOME=$THIRDPARTY_DIR/installed -export FLATBUFFERS_HOME=$THIRDPARTY_DIR/installed diff --git a/cpp/thirdparty/versions.sh b/cpp/thirdparty/versions.sh deleted file mode 100755 index a7b21e19fcc..00000000000 --- a/cpp/thirdparty/versions.sh +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. See accompanying LICENSE file. - -GTEST_VERSION=1.7.0 -GTEST_URL="https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz" -GTEST_BASEDIR=googletest-release-$GTEST_VERSION - -GBENCHMARK_VERSION=1.0.0 -GBENCHMARK_URL="https://github.com/google/benchmark/archive/v${GBENCHMARK_VERSION}.tar.gz" -GBENCHMARK_BASEDIR=benchmark-$GBENCHMARK_VERSION - -FLATBUFFERS_VERSION=1.3.0 -FLATBUFFERS_URL="https://github.com/google/flatbuffers/archive/v${FLATBUFFERS_VERSION}.tar.gz" -FLATBUFFERS_BASEDIR=flatbuffers-$FLATBUFFERS_VERSION From 7892baed7043d4225a1a62b38c755fb29e1bb9e2 Mon Sep 17 00:00:00 2001 From: Leif Walsh Date: Tue, 11 Oct 2016 06:23:23 -0400 Subject: [PATCH 3/4] ARROW-189: Fix darwin build. GTest's ExternalProject needs to set some CXXFLAGS on Darwin and there were quoting issues. --- cpp/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index df4c8bd17d1..d682dc76f8c 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -474,19 +474,19 @@ if(ARROW_BUILD_TESTS) if("$ENV{GTEST_HOME}" STREQUAL "") if(APPLE) - set(GTEST_CMAKE_ARGS "-DCMAKE_CXX_FLAGS=\"-fPIC -std=c++11 -stdlib=libc++ -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-value -Wno-ignored-attributes\"") + set(GTEST_CMAKE_CXX_FLAGS "-fPIC -std=c++11 -stdlib=libc++ -DGTEST_USE_OWN_TR1_TUPLE=1 -Wno-unused-value -Wno-ignored-attributes") else() - set(GTEST_CMAKE_ARGS "-DCMAKE_CXX_FLAGS=\"-fPIC\"") + set(GTEST_CMAKE_CXX_FLAGS "-fPIC") endif() ExternalProject_Add(googletest_ep URL "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz" - CMAKE_ARGS ${GTEST_CMAKE_ARGS} + CMAKE_ARGS -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS} # googletest doesn't define install rules, so just build in the # source dir and don't try to install. See its README for # details. BUILD_IN_SOURCE 1 - INSTALL_COMMAND "/bin/true") + INSTALL_COMMAND "") set(GTEST_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep") set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include") From e4fb63a1c925912c7c704e4700f49f9f16ea3365 Mon Sep 17 00:00:00 2001 From: Leif Walsh Date: Tue, 11 Oct 2016 06:28:19 -0400 Subject: [PATCH 4/4] ARROW-189: Remove 3rd party from conda build. --- cpp/conda.recipe/build.sh | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/cpp/conda.recipe/build.sh b/cpp/conda.recipe/build.sh index 6d7454e9272..0536fd99b5c 100644 --- a/cpp/conda.recipe/build.sh +++ b/cpp/conda.recipe/build.sh @@ -38,19 +38,9 @@ cd .. rm -rf conda-build mkdir conda-build - -cp -r thirdparty conda-build/ - cd conda-build pwd -# Build googletest for running unit tests -./thirdparty/download_thirdparty.sh -./thirdparty/build_thirdparty.sh gtest - -source thirdparty/versions.sh -export GTEST_HOME=`pwd`/thirdparty/$GTEST_BASEDIR - # if [ `uname` == Linux ]; then # SHARED_LINKER_FLAGS='-static-libstdc++' # elif [ `uname` == Darwin ]; then