Skip to content

add aws sdk to thirdparty #5234

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yangzhg merged 1 commit into from
Jan 20, 2021
Merged

add aws sdk to thirdparty #5234

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions be/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,30 @@ set_target_properties(orc PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/lib
add_library(cctz STATIC IMPORTED)
set_target_properties(cctz PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libcctz.a)

# add_library(aws-sdk-core STATIC IMPORTED)
# set_target_properties(aws-sdk-core PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-cpp-sdk-core.a)

# add_library(aws-sdk-s3 STATIC IMPORTED)
# set_target_properties(aws-sdk-s3 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-cpp-sdk-s3.a)

# add_library(aws-c-cal STATIC IMPORTED)
# set_target_properties(aws-c-cal PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-c-cal.a)

# add_library(aws-c-common STATIC IMPORTED)
# set_target_properties(aws-c-common PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-c-common.a)

# add_library(aws-c-event-stream STATIC IMPORTED)
# set_target_properties(aws-c-event-stream PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-c-event-stream.a)

# add_library(aws-c-io STATIC IMPORTED)
# set_target_properties(aws-c-io PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-c-io.a)

# add_library(aws-checksums STATIC IMPORTED)
# set_target_properties(aws-checksums PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-checksums.a)

# add_library(aws-s2n STATIC IMPORTED)
# set_target_properties(aws-s2n PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libs2n.a)

find_program(THRIFT_COMPILER thrift ${CMAKE_SOURCE_DIR}/bin)

# Check if functions are supported in this platform. All flags will generated
Expand Down Expand Up @@ -363,6 +387,8 @@ include_directories(
set(WL_START_GROUP "-Wl,--start-group")
set(WL_END_GROUP "-Wl,--end-group")

# set(AWS_LIBS aws-sdk-s3 aws-sdk-core aws-checksums aws-c-io aws-c-event-stream aws-c-common aws-c-cal aws-s2n)

# Set Palo libraries
set(DORIS_LINK_LIBS
${WL_START_GROUP}
Expand Down Expand Up @@ -428,6 +454,7 @@ set(DORIS_DEPENDENCIES
orc
odbc
cctz
# ${AWS_LIBS}
${WL_END_GROUP}
)

Expand Down
74 changes: 73 additions & 1 deletion thirdparty/build-thirdparty.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -710,6 +710,71 @@ build_tsan_header() {
cp $TSAN_HEADER_FILE $TP_INSTALL_DIR/include/sanitizer/
}

# aws-c-common
build_aws_c_common() {
check_if_source_exist $AWS_C_COMMON_SOURCE
cd $TP_SOURCE_DIR/$AWS_C_COMMON_SOURCE
mkdir -p $BUILD_DIR && cd $BUILD_DIR
cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF
${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install
}

# aws-c-event-stream
build_aws_c_event_stream() {
check_if_source_exist $AWS_C_EVENT_STREAM_SOURCE
cd $TP_SOURCE_DIR/$AWS_C_EVENT_STREAM_SOURCE
mkdir -p $BUILD_DIR && cd $BUILD_DIR
cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF \
-DBUILD_TESTING=OFF
${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install
}

# aws-checksums
build_aws_checksums() {
check_if_source_exist $AWS_CHECKSUMS_SOURCE
cd $TP_SOURCE_DIR/$AWS_CHECKSUMS_SOURCE
mkdir -p $BUILD_DIR && cd $BUILD_DIR
cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF
${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install
}

# aws-c-io
build_aws_c_io() {
check_if_source_exist $AWS_C_IO_SOURCE
cd $TP_SOURCE_DIR/$AWS_C_IO_SOURCE
mkdir -p $BUILD_DIR && cd $BUILD_DIR
cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF
${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install
}

# aws-s2n
build_aws_s2n() {
check_if_source_exist $AWS_S2N_SOURCE
cd $TP_SOURCE_DIR/$AWS_S2N_SOURCE
mkdir -p $BUILD_DIR && cd $BUILD_DIR
cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF
${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install
}

# aws-c-cal
build_aws_c_cal() {
check_if_source_exist $AWS_C_CAL_SOURCE
cd $TP_SOURCE_DIR/$AWS_C_CAL_SOURCE
mkdir -p $BUILD_DIR && cd $BUILD_DIR
cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF
${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install
}

# aws_sdk
build_aws_sdk() {
check_if_source_exist $AWS_SDK_SOURCE
cd $TP_SOURCE_DIR/$AWS_SDK_SOURCE
mkdir -p $BUILD_DIR && cd $BUILD_DIR
$CMAKE_CMD -G "${GENERATOR}" .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \
-DBUILD_DEPS=OFF -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF -DENABLE_TESTING=OFF \
-DCMAKE_MODULE_PATH=$TP_INSTALL_DIR/lib64/cmake -DBUILD_ONLY="s3"
${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install
}
# See https://github.com/apache/incubator-doris/issues/2910
# LLVM related codes have already be removed in master, so there is
# no need to build llvm tool here.
Expand Down Expand Up @@ -747,7 +812,14 @@ build_bitshuffle
build_croaringbitmap
build_orc
build_cctz
build_js_and_css
build_tsan_header
# build_aws_c_common
# build_aws_s2n
# build_aws_c_cal
# build_aws_c_io
# build_aws_checksums
# build_aws_c_event_stream
# build_aws_sdk
build_js_and_css

echo "Finihsed to build all thirdparties"
94 changes: 92 additions & 2 deletions thirdparty/vars.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -300,12 +300,102 @@ BOOTSTRAP_TABLE_CSS_NAME="bootstrap-table.min.css"
BOOTSTRAP_TABLE_CSS_FILE="bootstrap-table.min.css"
BOOTSTRAP_TABLE_CSS_MD5SUM="23389d4456da412e36bae30c469a766a"

# aws-c-common
AWS_C_COMMON_DOWNLOAD="https://github.com/awslabs/aws-c-common/archive/v0.4.63.tar.gz"
AWS_C_COMMON_NAME="aws-c-common-0.4.63.tar.gz"
AWS_C_COMMON_SOURCE="aws-c-common-0.4.63"
AWS_C_COMMON_MD5SUM="8298e00a0fb64779b7cf660592d50ab6"

# aws-c-event-stream
AWS_C_EVENT_STREAM_DOWNLOAD="https://github.com/awslabs/aws-c-event-stream/archive/v0.2.6.tar.gz"
AWS_C_EVENT_STREAM_NAME="aws-c-event-stream-0.2.6.tar.gz"
AWS_C_EVENT_STREAM_SOURCE="aws-c-event-stream-0.2.6"
AWS_C_EVENT_STREAM_MD5SUM="fceedde198ddbf38ffdaed08d1435f7f"

# aws-checksums
AWS_CHECKSUMS_DOWNLOAD="https://github.com/awslabs/aws-checksums/archive/v0.1.10.tar.gz"
AWS_CHECKSUMS_NAME="aws-checksums-0.1.10.tar.gz"
AWS_CHECKSUMS_SOURCE="aws-checksums-0.1.10"
AWS_CHECKSUMS_MD5SUM="2383c66f6250fa0238edbd1d779b49d3"

# aws-c-io
AWS_C_IO_DOWNLOAD="https://github.com/awslabs/aws-c-io/archive/v0.7.0.tar.gz"
AWS_C_IO_NAME="aws-c-io-0.7.0.tar.gz"
AWS_C_IO_SOURCE="aws-c-io-0.7.0"
AWS_C_IO_MD5SUM="b95a6f9d20500727231dd726c957276b"

# aws-s2n
AWS_S2N_DOWNLOAD="https://github.com/awslabs/s2n/archive/v0.10.0.tar.gz"
AWS_S2N_NAME="s2n-0.10.0.tar.gz"
AWS_S2N_SOURCE="s2n-0.10.0"
AWS_S2N_MD5SUM="9b3b39803b7090c2bd937f9cc73bc03f"

# aws-c-cal
AWS_C_CAL_DOWNLOAD="https://github.com/awslabs/aws-c-cal/archive/v0.4.5.tar.gz"
AWS_C_CAL_NAME="aws-c-cal-0.4.5.tar.gz"
AWS_C_CAL_SOURCE="aws-c-cal-0.4.5"
AWS_C_CAL_MD5SUM="317f3dbafae551a0fc7d70f31434e216"

# aws sdk
AWS_SDK_DOWNLOAD="https://github.com/aws/aws-sdk-cpp/archive/1.8.108.tar.gz"
AWS_SDK_NAME="aws-sdk-cpp-1.8.108.tar.gz"
AWS_SDK_SOURCE="aws-sdk-cpp-1.8.108"
AWS_SDK_MD5SUM="76d8855406e7da61f1f996c11c0b93d7"

# tsan_header
TSAN_HEADER_DOWNLOAD="https://gcc.gnu.org/git/?p=gcc.git;a=blob_plain;f=libsanitizer/include/sanitizer/tsan_interface_atomic.h;hb=refs/heads/releases/gcc-7"
TSAN_HEADER_NAME="tsan_interface_atomic.h"
TSAN_HEADER_FILE="tsan_interface_atomic.h"
TSAN_HEADER_MD5SUM="d72679bea167d6a513d959f5abd149dc"

# all thirdparties which need to be downloaded is set in array TP_ARCHIVES
export TP_ARCHIVES="LIBEVENT OPENSSL THRIFT LLVM CLANG COMPILER_RT PROTOBUF GFLAGS GLOG GTEST RAPIDJSON SNAPPY GPERFTOOLS ZLIB LZ4 BZIP LZO2 CURL RE2 BOOST MYSQL BOOST_FOR_MYSQL ODBC LEVELDB BRPC ROCKSDB LIBRDKAFKA FLATBUFFERS ARROW BROTLI DOUBLE_CONVERSION ZSTD S2 BITSHUFFLE CROARINGBITMAP ORC JEMALLOC CCTZ DATATABLES BOOTSTRAP_TABLE_JS BOOTSTRAP_TABLE_CSS TSAN_HEADER"

export TP_ARCHIVES="LIBEVENT
OPENSSL
THRIFT
LLVM
CLANG
COMPILER_RT
PROTOBUF
GFLAGS
GLOG
GTEST
RAPIDJSON
SNAPPY
GPERFTOOLS
ZLIB
LZ4
BZIP
LZO2
CURL
RE2
BOOST
MYSQL
BOOST_FOR_MYSQL
ODBC
LEVELDB
BRPC
ROCKSDB
LIBRDKAFKA
FLATBUFFERS
ARROW
BROTLI
DOUBLE_CONVERSION
ZSTD
S2
BITSHUFFLE
CROARINGBITMAP
ORC
JEMALLOC
CCTZ
DATATABLES
BOOTSTRAP_TABLE_JS
BOOTSTRAP_TABLE_CSS
TSAN_HEADER
AWS_C_COMMON
AWS_C_EVENT_STREAM
AWS_C_IO
AWS_C_CAL
AWS_C_IO
AWS_CHECKSUMS
AWS_S2N
AWS_SDK"