[Feature] libhdfs3 built enable kerberos support

be/CMakeLists.txt

@@ -85,6 +85,7 @@ set(BASE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(ENV{DORIS_HOME} "${BASE_DIR}/../")
 set(BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(THIRDPARTY_DIR "$ENV{DORIS_THIRDPARTY}/installed/")
+set(HDFS3_KRB5_INSTALL_DIR "$ENV{DORIS_THIRDPARTY}/installed/libhdfs_with_kerberos/")
 set(GENSRC_DIR "${BASE_DIR}/../gensrc/build/")
 set(SRC_DIR "${BASE_DIR}/src/")
 set(TEST_DIR "${CMAKE_SOURCE_DIR}/test/")
@@ -97,6 +98,7 @@ else()
 endif()
 message(STATUS "make test: ${MAKE_TEST}")
 option(WITH_MYSQL "Support access MySQL" ON)
+option(WITH_KERBEROS "Enable or disable Kereberos support" ${WITH_KERBEROS})
 
 # Check gcc
 if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
@@ -328,17 +330,40 @@ set_target_properties(minizip PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib
 add_library(idn STATIC IMPORTED)
 set_target_properties(idn PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libidn.a)
 
-add_library(gsasl STATIC IMPORTED)
-set_target_properties(gsasl PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libgsasl.a)
-
 add_library(breakpad STATIC IMPORTED)
 set_target_properties(breakpad PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libbreakpad_client.a)
 
-if (ARCH_AMD64)
-    # libhdfs3 only support x86 or amd64
+if (WITH_KERBEROS)
+    # kerberos lib for libhdfs3 
+    add_library(gsasl STATIC IMPORTED)
+    set_target_properties(gsasl PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libgsasl.a)
+
+    add_library(krb5support STATIC IMPORTED)
+    set_target_properties(krb5support PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libkrb5support.a)
+
+    add_library(krb5 STATIC IMPORTED)
+    set_target_properties(krb5 PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libkrb5.a)
+
+    add_library(com_err STATIC IMPORTED)
+    set_target_properties(com_err PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libcom_err.a)
+
+    add_library(k5crypto STATIC IMPORTED)
+    set_target_properties(k5crypto PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libk5crypto.a)
+
+    add_library(gssapi_krb5 STATIC IMPORTED)
+    set_target_properties(gssapi_krb5 PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libgssapi_krb5.a)
+
+    add_library(hdfs3 STATIC IMPORTED)
+    set_target_properties(hdfs3 PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libhdfs3.a)
+else()
+    add_library(gsasl STATIC IMPORTED)
+    set_target_properties(gsasl PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libgsasl.a)
+
     add_library(hdfs3 STATIC IMPORTED)
     set_target_properties(hdfs3 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libhdfs3.a)
+endif()
 
+if (ARCH_AMD64)
     add_library(xml2 STATIC IMPORTED)
     set_target_properties(xml2 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libxml2.a)
 
@@ -507,6 +532,14 @@ endif()
 set(WL_START_GROUP "-Wl,--start-group")
 set(WL_END_GROUP "-Wl,--end-group")
 
+
+set(KRB5_LIBS
+    krb5support
+    krb5
+    com_err
+    gssapi_krb5
+    k5crypto)
+
 set(AWS_LIBS
     aws-sdk-s3
     aws-sdk-core
@@ -640,6 +673,12 @@ if (WITH_MYSQL)
         )
 endif()
 
+if (WITH_KERBEROS)
+    set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES}
+    	${KRB5_LIBS}
+        )
+endif()
+
 set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} ${WL_END_GROUP})
 
 message(STATUS "DORIS_DEPENDENCIES is ${DORIS_DEPENDENCIES}")
@@ -651,6 +690,7 @@ set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS}
     -static-libstdc++
     -static-libgcc
     -lstdc++fs
+    -lresolv
 )
 
 if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")

build.sh

@@ -175,7 +175,9 @@ if [[ ${HELP} -eq 1 ]]; then
     usage
     exit
 fi
-
+if [[ -z ${WITH_KERBEROS} ]]; then
+    WITH_KERBEROS=ON
+fi
 # build thirdparty libraries if necessary
 if [[ ! -f ${DORIS_THIRDPARTY}/installed/lib/libbacktrace.a ]]; then
     echo "Thirdparty libraries need to be build ..."
@@ -228,6 +230,7 @@ echo "Get params:
     PARALLEL            -- $PARALLEL
     CLEAN               -- $CLEAN
     WITH_MYSQL          -- $WITH_MYSQL
+    WITH_KERBEROS       -- $WITH_KERBEROS
     WITH_LZO            -- $WITH_LZO
     GLIBC_COMPATIBILITY -- $GLIBC_COMPATIBILITY
     USE_AVX2            -- $USE_AVX2
@@ -289,6 +292,7 @@ if [ ${BUILD_BE} -eq 1 ] ; then
             -DMAKE_TEST=OFF \
             ${CMAKE_USE_CCACHE} \
             -DWITH_MYSQL=${WITH_MYSQL} \
+            -DWITH_KERBEROS=${WITH_KERBEROS} \
             -DWITH_LZO=${WITH_LZO} \
             -DUSE_LIBCPP=${USE_LIBCPP} \
             -DBUILD_META_TOOL=${BUILD_META_TOOL} \

run-be-ut.sh

@@ -136,6 +136,7 @@ ${CMAKE_CMD} -G "${GENERATOR}" \
     -DGLIBC_COMPATIBILITY="${GLIBC_COMPATIBILITY}" \
     -DBUILD_META_TOOL=OFF \
     -DWITH_MYSQL=OFF \
+    -DWITH_KERBEROS=OFF \
     -DUSE_DWARF=${USE_DWARF} \
     ${CMAKE_USE_CCACHE} ../
 ${BUILD_SYSTEM} -j ${PARALLEL}

thirdparty/build-thirdparty.sh

@@ -904,16 +904,46 @@ build_gsasl() {
     make -j $PARALLEL && make install
 }
 
+# build_gsasl2 just for libgsasl1.8.0
+build_gsasl2() {
+    check_if_source_exist $GSASL2_SOURCE
+    cd $TP_SOURCE_DIR/$GSASL2_SOURCE
+    mkdir -p $BUILD_DIR && cd $BUILD_DIR
+    ../configure --prefix=$HDFS3_KRB5_INSTALL_DIR --with-gssapi-impl=mit --enable-shared=no --with-pic --with-libidn-prefix=$TP_INSTALL_DIR
+    make -j $PARALLEL && make install
+}
+
+# krb5
+build_krb5() {
+    check_if_source_exist $KRB5_SOURCE
+    cd $TP_SOURCE_DIR/$KRB5_SOURCE/src
+    mkdir -p $BUILD_DIR && cd $BUILD_DIR
+    CFLAGS="-fcommon" \
+    ../configure --prefix=$HDFS3_KRB5_INSTALL_DIR --disable-shared --enable-static
+    make -j $PARALLEL && make install
+}
+
 # hdfs3
 build_hdfs3() {
     check_if_source_exist $HDFS3_SOURCE
     cd $TP_SOURCE_DIR/$HDFS3_SOURCE
-    mkdir -p $BUILD_DIR && cd $BUILD_DIR
-    ../bootstrap --dependency=$TP_INSTALL_DIR --prefix=$TP_INSTALL_DIR
+    mkdir -p $BUILD_DIR && cd $BUILD_DIR && rm ./* -rf
+    # build libhdfs3 without kerberos
+    ../bootstrap --dependency="$TP_INSTALL_DIR" --prefix=$TP_INSTALL_DIR
     make CXXFLAGS="$libhdfs_cxx17" -j $PARALLEL
     make install
 }
 
+# hdfs3_with_kerberos
+build_hdfs3_with_kerberos() {
+    check_if_source_exist $HDFS3_SOURCE
+    cd $TP_SOURCE_DIR/$HDFS3_SOURCE
+    mkdir -p $BUILD_DIR && cd $BUILD_DIR && rm ./* -rf
+    # build libhdfs3 with kerberos support
+    ../bootstrap --dependency="$HDFS3_KRB5_INSTALL_DIR:$TP_INSTALL_DIR -DWITH_KERBEROS=true" --prefix=$HDFS3_KRB5_INSTALL_DIR
+    make CXXFLAGS="$libhdfs_cxx17" -j $PARALLEL
+    make install
+}
 # benchmark
 build_benchmark() {
     check_if_source_exist $BENCHMARK_SOURCE
@@ -999,7 +1029,10 @@ build_lzma
 build_xml2
 build_idn
 build_gsasl
+build_gsasl2
+build_krb5
 build_hdfs3
+build_hdfs3_with_kerberos
 build_benchmark
 build_breakpad
 build_simdjson

thirdparty/download-thirdparty.sh

@@ -156,7 +156,7 @@ do
 done
 echo "===== Downloading thirdparty archives...done"
 
-# check if all tp archievs exists
+# check if all tp archives exists
 echo "===== Checking all thirdpart archives..."
 for TP_ARCH in ${TP_ARCHIVES[*]}
 do
@@ -271,6 +271,16 @@ fi
 cd -
 echo "Finished patching $S2_SOURCE"
 
+# gsasl2 patch to fix link error such as mutilple func defination
+# when link target with kerberos
+cd $TP_SOURCE_DIR/$GSASL2_SOURCE
+if [ ! -f $PATCHED_MARK ]; then
+    patch -p1 < $TP_PATCH_DIR/libgsasl-1.8.0.patch
+    touch $PATCHED_MARK
+fi
+cd -
+echo "Finished patching $GSASL2_SOURCE"
+
 # hdfs3 patch to fix compile error
 cd $TP_SOURCE_DIR/$HDFS3_SOURCE
 if [ ! -f $PATCHED_MARK ]; then

thirdparty/patches/libgsasl-1.8.0.patch

@@ -0,0 +1,24 @@
+--- a/config.h.in	2012-05-29 01:11:28.000000000 +0800
++++ b/config.h.in	2022-05-11 18:06:02.000000000 +0800
+@@ -176,7 +176,7 @@
+ #undef HAVE_GSSAPI_H
+
+ /* Define to 1 if you have the `GSS_C_NT_HOSTBASED_SERVICE' function. */
+-#undef HAVE_GSS_C_NT_HOSTBASED_SERVICE
++#define HAVE_GSS_C_NT_HOSTBASED_SERVICE 1
+
+ /* Define to 1 if you have the `gss_decapsulate_token' function. */
+ #undef HAVE_GSS_DECAPSULATE_TOKEN
+@@ -185,10 +185,10 @@
+ #undef HAVE_GSS_ENCAPSULATE_TOKEN
+
+ /* Define to 1 if you have the `gss_inquire_mech_for_saslname' function. */
+-#undef HAVE_GSS_INQUIRE_MECH_FOR_SASLNAME
++#define HAVE_GSS_INQUIRE_MECH_FOR_SASLNAME 1
+
+ /* Define to 1 if you have the `gss_oid_equal' function. */
+-#undef HAVE_GSS_OID_EQUAL
++#define HAVE_GSS_OID_EQUAL 1
+
+ /* Define if you have the iconv() function and it works. */
+ #undef HAVE_ICONV

thirdparty/vars.sh

@@ -32,6 +32,9 @@ export TP_SOURCE_DIR=$TP_DIR/src
 # thirdparties will be installed to here
 export TP_INSTALL_DIR=$TP_DIR/installed
 
+# libhdfs3-with-kerberos will be installed to here
+export HDFS3_KRB5_INSTALL_DIR=$TP_INSTALL_DIR/libhdfs_with_kerberos
+
 # patches for all thirdparties
 export TP_PATCH_DIR=$TP_DIR/patches
 
@@ -44,8 +47,8 @@ export TP_LIB_DIR=$TP_INSTALL_DIR/lib
 # all java libraries will be unpacked to here
 export TP_JAR_DIR=$TP_INSTALL_DIR/lib/jar
 
-# source of all dependencies
-export REPOSITORY_URL=https://doris-thirdparty-repo.bj.bcebos.com/thirdparty
+# source of all dependencies, default unuse it
+# export REPOSITORY_URL=https://doris-thirdparty-repo.bj.bcebos.com/thirdparty
 
 #####################################################
 # Download url, filename and unpaced filename
@@ -326,6 +329,17 @@ GSASL_NAME="libgsasl-1.10.0.tar.gz"
 GSASL_SOURCE="libgsasl-1.10.0"
 GSASL_MD5SUM="9c8fc632da4ce108fb7581b33de2a5ce"
 
+GSASL2_DOWNLOAD="https://ftp.gnu.org/gnu/gsasl/libgsasl-1.8.0.tar.gz"
+GSASL2_NAME="libgsasl-1.8.0.tar.gz"
+GSASL2_SOURCE="libgsasl-1.8.0"
+GSASL2_MD5SUM="5dbdf859f6e60e05813370e2b193b92b"
+
+# krb5
+KRB5_DOWNLOAD="https://kerberos.org/dist/krb5/1.19/krb5-1.19.tar.gz"
+KRB5_NAME="krb5-1.19.tar.gz"
+KRB5_SOURCE="krb5-1.19"
+KRB5_MD5SUM="aaf18447a5a014aa3b7e81814923f4c9"
+
 # hdfs3
 HDFS3_DOWNLOAD="https://doris-thirdparty-repo.bj.bcebos.com/thirdparty/libhdfs3-master.zip"
 HDFS3_NAME="libhdfs3-master.zip"
@@ -423,6 +437,8 @@ LZMA
 XML2
 IDN
 GSASL
+GSASL2
+KRB5
 HDFS3
 LIBDIVIDE
 PDQSORT