diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index a9d0c6b5574ae9..7ba593fe2f5881 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -240,6 +240,30 @@ set_target_properties(orc PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/lib add_library(cctz STATIC IMPORTED) set_target_properties(cctz PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libcctz.a) +# add_library(aws-sdk-core STATIC IMPORTED) +# set_target_properties(aws-sdk-core PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-cpp-sdk-core.a) + +# add_library(aws-sdk-s3 STATIC IMPORTED) +# set_target_properties(aws-sdk-s3 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-cpp-sdk-s3.a) + +# add_library(aws-c-cal STATIC IMPORTED) +# set_target_properties(aws-c-cal PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-c-cal.a) + +# add_library(aws-c-common STATIC IMPORTED) +# set_target_properties(aws-c-common PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-c-common.a) + +# add_library(aws-c-event-stream STATIC IMPORTED) +# set_target_properties(aws-c-event-stream PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-c-event-stream.a) + +# add_library(aws-c-io STATIC IMPORTED) +# set_target_properties(aws-c-io PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-c-io.a) + +# add_library(aws-checksums STATIC IMPORTED) +# set_target_properties(aws-checksums PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-checksums.a) + +# add_library(aws-s2n STATIC IMPORTED) +# set_target_properties(aws-s2n PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libs2n.a) + find_program(THRIFT_COMPILER thrift ${CMAKE_SOURCE_DIR}/bin) # Check if functions are supported in this platform. All flags will generated @@ -363,6 +387,8 @@ include_directories( set(WL_START_GROUP "-Wl,--start-group") set(WL_END_GROUP "-Wl,--end-group") +# set(AWS_LIBS aws-sdk-s3 aws-sdk-core aws-checksums aws-c-io aws-c-event-stream aws-c-common aws-c-cal aws-s2n) + # Set Palo libraries set(DORIS_LINK_LIBS ${WL_START_GROUP} @@ -428,6 +454,7 @@ set(DORIS_DEPENDENCIES orc odbc cctz + # ${AWS_LIBS} ${WL_END_GROUP} ) diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index d22407040b3582..a13ce3fe60d6b5 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -710,6 +710,71 @@ build_tsan_header() { cp $TSAN_HEADER_FILE $TP_INSTALL_DIR/include/sanitizer/ } +# aws-c-common +build_aws_c_common() { + check_if_source_exist $AWS_C_COMMON_SOURCE + cd $TP_SOURCE_DIR/$AWS_C_COMMON_SOURCE + mkdir -p $BUILD_DIR && cd $BUILD_DIR + cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF + ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install +} + +# aws-c-event-stream +build_aws_c_event_stream() { + check_if_source_exist $AWS_C_EVENT_STREAM_SOURCE + cd $TP_SOURCE_DIR/$AWS_C_EVENT_STREAM_SOURCE + mkdir -p $BUILD_DIR && cd $BUILD_DIR + cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_TESTING=OFF + ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install +} + +# aws-checksums +build_aws_checksums() { + check_if_source_exist $AWS_CHECKSUMS_SOURCE + cd $TP_SOURCE_DIR/$AWS_CHECKSUMS_SOURCE + mkdir -p $BUILD_DIR && cd $BUILD_DIR + cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF + ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install +} + +# aws-c-io +build_aws_c_io() { + check_if_source_exist $AWS_C_IO_SOURCE + cd $TP_SOURCE_DIR/$AWS_C_IO_SOURCE + mkdir -p $BUILD_DIR && cd $BUILD_DIR + cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF + ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install +} + +# aws-s2n +build_aws_s2n() { + check_if_source_exist $AWS_S2N_SOURCE + cd $TP_SOURCE_DIR/$AWS_S2N_SOURCE + mkdir -p $BUILD_DIR && cd $BUILD_DIR + cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF + ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install +} + +# aws-c-cal +build_aws_c_cal() { + check_if_source_exist $AWS_C_CAL_SOURCE + cd $TP_SOURCE_DIR/$AWS_C_CAL_SOURCE + mkdir -p $BUILD_DIR && cd $BUILD_DIR + cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF + ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install +} + +# aws_sdk +build_aws_sdk() { + check_if_source_exist $AWS_SDK_SOURCE + cd $TP_SOURCE_DIR/$AWS_SDK_SOURCE + mkdir -p $BUILD_DIR && cd $BUILD_DIR + $CMAKE_CMD -G "${GENERATOR}" .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \ + -DBUILD_DEPS=OFF -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF -DENABLE_TESTING=OFF \ + -DCMAKE_MODULE_PATH=$TP_INSTALL_DIR/lib64/cmake -DBUILD_ONLY="s3" + ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install +} # See https://github.com/apache/incubator-doris/issues/2910 # LLVM related codes have already be removed in master, so there is # no need to build llvm tool here. @@ -747,7 +812,14 @@ build_bitshuffle build_croaringbitmap build_orc build_cctz -build_js_and_css build_tsan_header +# build_aws_c_common +# build_aws_s2n +# build_aws_c_cal +# build_aws_c_io +# build_aws_checksums +# build_aws_c_event_stream +# build_aws_sdk +build_js_and_css echo "Finihsed to build all thirdparties" diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 020bd556713683..6f8dc123db24b4 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -300,6 +300,48 @@ BOOTSTRAP_TABLE_CSS_NAME="bootstrap-table.min.css" BOOTSTRAP_TABLE_CSS_FILE="bootstrap-table.min.css" BOOTSTRAP_TABLE_CSS_MD5SUM="23389d4456da412e36bae30c469a766a" +# aws-c-common +AWS_C_COMMON_DOWNLOAD="https://github.com/awslabs/aws-c-common/archive/v0.4.63.tar.gz" +AWS_C_COMMON_NAME="aws-c-common-0.4.63.tar.gz" +AWS_C_COMMON_SOURCE="aws-c-common-0.4.63" +AWS_C_COMMON_MD5SUM="8298e00a0fb64779b7cf660592d50ab6" + +# aws-c-event-stream +AWS_C_EVENT_STREAM_DOWNLOAD="https://github.com/awslabs/aws-c-event-stream/archive/v0.2.6.tar.gz" +AWS_C_EVENT_STREAM_NAME="aws-c-event-stream-0.2.6.tar.gz" +AWS_C_EVENT_STREAM_SOURCE="aws-c-event-stream-0.2.6" +AWS_C_EVENT_STREAM_MD5SUM="fceedde198ddbf38ffdaed08d1435f7f" + +# aws-checksums +AWS_CHECKSUMS_DOWNLOAD="https://github.com/awslabs/aws-checksums/archive/v0.1.10.tar.gz" +AWS_CHECKSUMS_NAME="aws-checksums-0.1.10.tar.gz" +AWS_CHECKSUMS_SOURCE="aws-checksums-0.1.10" +AWS_CHECKSUMS_MD5SUM="2383c66f6250fa0238edbd1d779b49d3" + +# aws-c-io +AWS_C_IO_DOWNLOAD="https://github.com/awslabs/aws-c-io/archive/v0.7.0.tar.gz" +AWS_C_IO_NAME="aws-c-io-0.7.0.tar.gz" +AWS_C_IO_SOURCE="aws-c-io-0.7.0" +AWS_C_IO_MD5SUM="b95a6f9d20500727231dd726c957276b" + +# aws-s2n +AWS_S2N_DOWNLOAD="https://github.com/awslabs/s2n/archive/v0.10.0.tar.gz" +AWS_S2N_NAME="s2n-0.10.0.tar.gz" +AWS_S2N_SOURCE="s2n-0.10.0" +AWS_S2N_MD5SUM="9b3b39803b7090c2bd937f9cc73bc03f" + +# aws-c-cal +AWS_C_CAL_DOWNLOAD="https://github.com/awslabs/aws-c-cal/archive/v0.4.5.tar.gz" +AWS_C_CAL_NAME="aws-c-cal-0.4.5.tar.gz" +AWS_C_CAL_SOURCE="aws-c-cal-0.4.5" +AWS_C_CAL_MD5SUM="317f3dbafae551a0fc7d70f31434e216" + +# aws sdk +AWS_SDK_DOWNLOAD="https://github.com/aws/aws-sdk-cpp/archive/1.8.108.tar.gz" +AWS_SDK_NAME="aws-sdk-cpp-1.8.108.tar.gz" +AWS_SDK_SOURCE="aws-sdk-cpp-1.8.108" +AWS_SDK_MD5SUM="76d8855406e7da61f1f996c11c0b93d7" + # tsan_header TSAN_HEADER_DOWNLOAD="https://gcc.gnu.org/git/?p=gcc.git;a=blob_plain;f=libsanitizer/include/sanitizer/tsan_interface_atomic.h;hb=refs/heads/releases/gcc-7" TSAN_HEADER_NAME="tsan_interface_atomic.h" @@ -307,5 +349,53 @@ TSAN_HEADER_FILE="tsan_interface_atomic.h" TSAN_HEADER_MD5SUM="d72679bea167d6a513d959f5abd149dc" # all thirdparties which need to be downloaded is set in array TP_ARCHIVES -export TP_ARCHIVES="LIBEVENT OPENSSL THRIFT LLVM CLANG COMPILER_RT PROTOBUF GFLAGS GLOG GTEST RAPIDJSON SNAPPY GPERFTOOLS ZLIB LZ4 BZIP LZO2 CURL RE2 BOOST MYSQL BOOST_FOR_MYSQL ODBC LEVELDB BRPC ROCKSDB LIBRDKAFKA FLATBUFFERS ARROW BROTLI DOUBLE_CONVERSION ZSTD S2 BITSHUFFLE CROARINGBITMAP ORC JEMALLOC CCTZ DATATABLES BOOTSTRAP_TABLE_JS BOOTSTRAP_TABLE_CSS TSAN_HEADER" - +export TP_ARCHIVES="LIBEVENT +OPENSSL +THRIFT +LLVM +CLANG +COMPILER_RT +PROTOBUF +GFLAGS +GLOG +GTEST +RAPIDJSON +SNAPPY +GPERFTOOLS +ZLIB +LZ4 +BZIP +LZO2 +CURL +RE2 +BOOST +MYSQL +BOOST_FOR_MYSQL +ODBC +LEVELDB +BRPC +ROCKSDB +LIBRDKAFKA +FLATBUFFERS +ARROW +BROTLI +DOUBLE_CONVERSION +ZSTD +S2 +BITSHUFFLE +CROARINGBITMAP +ORC +JEMALLOC +CCTZ +DATATABLES +BOOTSTRAP_TABLE_JS +BOOTSTRAP_TABLE_CSS +TSAN_HEADER +AWS_C_COMMON +AWS_C_EVENT_STREAM +AWS_C_IO +AWS_C_CAL +AWS_C_IO +AWS_CHECKSUMS +AWS_S2N +AWS_SDK"