[selectdb] add build release scripts (#2)

ci: align build-for-release-selectdb.sh script (#10) [release](build script) fix build release bug when build multiple version on same machine #32104 change jdk path to open jdk (#108)
apache · Mar 14, 2024 · 84d152b · 84d152b
1 parent 5273cbe
commit 84d152b
Show file tree

Hide file tree

Showing 9 changed files with 223 additions and 4 deletions.
diff --git a/bin/start_be.sh b/bin/start_be.sh
@@ -63,6 +63,15 @@ DORIS_HOME="$(
     pwd
 )"
 export DORIS_HOME
+if [[ -z "${JAVA_HOME}" ]]; then
+    java_home="$(
+       cd "${curdir}/../.."
+       pwd
+    )"
+    export JAVA_HOME=$java_home/java8
+    export PATH=$JAVA_HOME/bin:$PATH
+    echo "${JAVA_HOME}"
+fi
 
 if [[ "$(uname -s)" != 'Darwin' ]]; then
     MAX_MAP_COUNT="$(cat /proc/sys/vm/max_map_count)"

diff --git a/bin/start_fe.sh b/bin/start_fe.sh
@@ -89,6 +89,17 @@ DORIS_HOME="$(
 )"
 export DORIS_HOME
 
+if [[ -z "${JAVA_HOME}" ]]; then
+    java_home="$(
+       cd "${curdir}/../.."
+       pwd
+    )"
+    export JAVA_HOME=$java_home/java8
+    export PATH=$JAVA_HOME/bin:$PATH
+    echo "${JAVA_HOME}"
+fi
+
+
 # export env variables from fe.conf
 #
 # JAVA_OPTS

diff --git a/build-for-release-selectdb.sh b/build-for-release-selectdb.sh
@@ -0,0 +1,183 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+##############################################################
+# This script is used to build for SelectDB Enterprise Core
+##############################################################
+
+set -eo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+
+export DORIS_HOME="${ROOT}"
+
+# Check args
+usage() {
+    echo "
+Usage: $0 --version version <options>
+  Optional options:
+     [no option]        build with avx2
+     --noavx2           build without avx2
+     --tar              pack the output
+
+  Eg.
+    $0 --version 1.2.0                      build with avx2
+    $0 --noavx2 --version 1.2.0             build without avx2
+    $0 --version 1.2.0 --tar                build with avx2 and pack the output
+  "
+    exit 1
+}
+
+if ! OPTS="$(getopt \
+    -n "$0" \
+    -o '' \
+    -l 'noavx2' \
+    -l 'tar' \
+    -l 'version:' \
+    -l 'help' \
+    -- "$@")"; then
+    usage
+fi
+
+eval set -- "${OPTS}"
+
+_USE_AVX2=1
+TAR=0
+VERSION=
+if [[ "$#" == 1 ]]; then
+    _USE_AVX2=1
+else
+    while true; do
+        case "$1" in
+        --noavx2)
+            _USE_AVX2=0
+            shift
+            ;;
+        --tar)
+            TAR=1
+            shift
+            ;;
+        --version)
+            VERSION="$2"
+            shift 2
+            ;;
+        --help)
+            HELP=1
+            shift
+            ;;
+        --)
+            shift
+            break
+            ;;
+        *)
+            echo "Internal error"
+            exit 1
+            ;;
+        esac
+    done
+fi
+
+if [[ "${HELP}" -eq 1 ]]; then
+    usage
+    exit
+fi
+
+if [[ -z ${VERSION} ]]; then
+    echo "Must specify version"
+    usage
+    exit 1
+fi
+
+echo "Get params:
+    VERSION         -- ${VERSION}
+    USE_AVX2        -- ${_USE_AVX2}
+    TAR             -- ${TAR}
+"
+
+ARCH="$(uname -m)"
+if [[ "${ARCH}" == "aarch64" ]]; then
+    ARCH="arm64"
+elif [[ "${ARCH}" == "x86_64" ]]; then
+    ARCH="x64"
+else
+    echo "Unknown arch: ${ARCH}"
+    exit 1
+fi
+
+ORI_OUTPUT="${ROOT}/output"
+
+FE="fe"
+BE="be"
+BROKER="apache_hdfs_broker"
+AUDIT="audit_loader"
+
+PACKAGE_NAME="selectdb-doris-${VERSION}-bin-${ARCH}"
+if [[ "${_USE_AVX2}" == "0" && "${ARCH}" == "x64" ]]; then
+    PACKAGE_NAME="${PACKAGE_NAME}-noavx2"
+fi
+OUTPUT="${ORI_OUTPUT}/${PACKAGE_NAME}"
+
+rm -rf "${OUTPUT}" && mkdir -p "${OUTPUT}"
+echo "Package Path: ${OUTPUT}"
+
+# download and setup java
+JAVA8_DOWNLOAD_LINK=
+JAVA8_DIR_NAME=
+if [[ "${ARCH}" == "x64" ]]; then
+    JAVA8_DOWNLOAD_LINK="https://selectdb-doris-1308700295.cos.ap-beijing.myqcloud.com/release/jdbc_driver/openjdk-8u352-b08-linux-x64.tar.gz"
+    JAVA8_DIR_NAME="openjdk-8u352-b08-linux-x64"
+elif [[ "${ARCH}" == "arm64" ]]; then
+    JAVA8_DOWNLOAD_LINK="https://selectdb-doris-1308700295.cos.ap-beijing.myqcloud.com/release/jdbc_driver/bisheng-jdk-8u352-linux-aarch64.tar.gz"
+    JAVA8_DIR_NAME="bisheng-jdk1.8.0_352"
+else
+    echo "Unknown arch: ${ARCH}"
+    exit 1
+fi
+
+OUTPUT_JAVA8="${OUTPUT}/java8"
+curl -# "${JAVA8_DOWNLOAD_LINK}" | tar xz -C "${OUTPUT}/" && mv "${OUTPUT}/${JAVA8_DIR_NAME}/" "${OUTPUT_JAVA8}"
+export JAVA_HOME="${OUTPUT_JAVA8}"
+export PATH="${JAVA_HOME}/bin:${PATH}"
+
+# build core
+#sh build.sh --clean &&
+USE_AVX2="${_USE_AVX2}" sh build.sh && USE_AVX2="${_USE_AVX2}" sh build.sh --be --meta-tool
+
+echo "Begin to install"
+cp -r "${ORI_OUTPUT}/fe" "${OUTPUT}/fe"
+cp -r "${ORI_OUTPUT}/be" "${OUTPUT}/be"
+cp -r "${ORI_OUTPUT}/apache_hdfs_broker" "${OUTPUT}/apache_hdfs_broker"
+cp -r "${ORI_OUTPUT}/audit_loader" "${OUTPUT}/audit_loader"
+
+JDBC_DRIVERS_DIR="${OUTPUT}/jdbc_drivers/"
+mkdir -p "${JDBC_DRIVERS_DIR}"
+wget https://selectdb-doris-1308700295.cos.ap-beijing.myqcloud.com/release/jdbc_driver/clickhouse-jdbc-0.3.2-patch11-all.jar -P "${JDBC_DRIVERS_DIR}/"
+wget https://selectdb-doris-1308700295.cos.ap-beijing.myqcloud.com/release/jdbc_driver/mssql-jdbc-11.2.0.jre8.jar -P "${JDBC_DRIVERS_DIR}/"
+wget https://selectdb-doris-1308700295.cos.ap-beijing.myqcloud.com/release/jdbc_driver/mysql-connector-java-8.0.25.jar -P "${JDBC_DRIVERS_DIR}/"
+wget https://selectdb-doris-1308700295.cos.ap-beijing.myqcloud.com/release/jdbc_driver/ojdbc6.jar -P "${JDBC_DRIVERS_DIR}/"
+wget https://selectdb-doris-1308700295.cos.ap-beijing.myqcloud.com/release/jdbc_driver/postgresql-42.5.0.jar -P "${JDBC_DRIVERS_DIR}/"
+
+if [[ "${TAR}" -eq 1 ]]; then
+    echo "Begin to compress"
+    cd "${ORI_OUTPUT}"
+    tar -cf - "${PACKAGE_NAME}" | xz -T0 -z - >"${PACKAGE_NAME}".tar.xz
+    cd -
+fi
+
+echo "Output dir: ${OUTPUT}"
+exit 0
diff --git a/build-for-release.sh b/build-for-release.sh
@@ -122,6 +122,7 @@ fi
 echo "ARCH: ${ARCH}"
 
 ORI_OUTPUT="${ROOT}/output"
+rm -rf "${ORI_OUTPUT}"
 
 FE="fe"
 BE="be"

diff --git a/conf/be.conf b/conf/be.conf
@@ -53,6 +53,9 @@ ssl_certificate_path = "$DORIS_HOME/conf/cert.pem"
 ssl_private_key_path = "$DORIS_HOME/conf/key.pem"
 
 
+# jdbc driver path configuration
+jdbc_drivers_dir=${DORIS_HOME}/../jdbc_drivers/
+
 # Choose one if there are more than one ip except loopback address. 
 # Note that there should at most one ip match this list.
 # If no ip match this rule, will choose one randomly.

diff --git a/conf/fe.conf b/conf/fe.conf
@@ -64,6 +64,9 @@ query_port = 9030
 edit_log_port = 9010
 arrow_flight_sql_port = -1
 
+# jdbc driver path configuration
+jdbc_drivers_dir=${DORIS_HOME}/../jdbc_drivers/
+
 # Choose one if there are more than one ip except loopback address. 
 # Note that there should at most one ip match this list.
 # If no ip match this rule, will choose one randomly.

diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/GlobalVariable.java
@@ -57,8 +57,8 @@ public final class GlobalVariable {
     public static final String AUDIT_PLUGIN_MAX_SQL_LENGTH = "audit_plugin_max_sql_length";
 
     @VariableMgr.VarAttr(name = VERSION_COMMENT, flag = VariableMgr.READ_ONLY)
-    public static String versionComment = "Doris version "
-            + Version.DORIS_BUILD_VERSION + "-" + Version.DORIS_BUILD_SHORT_HASH;
+    public static String versionComment =
+            Version.DORIS_BUILD_VERSION + "-" + Version.DORIS_BUILD_SHORT_HASH;
 
     @VariableMgr.VarAttr(name = VERSION, flag = VariableMgr.READ_ONLY)
     public static String version = MysqlHandshakePacket.SERVER_VERSION;

diff --git a/fs_brokers/apache_hdfs_broker/bin/start_broker.sh b/fs_brokers/apache_hdfs_broker/bin/start_broker.sh
@@ -56,6 +56,15 @@ BROKER_HOME="$(
     pwd
 )"
 export BROKER_HOME
+if [[ -z "${JAVA_HOME}" ]]; then
+    java_home="$(
+       cd "${curdir}/../.."
+       pwd
+    )"
+    export JAVA_HOME=$java_home/java8
+    export PATH=$JAVA_HOME/bin:$PATH
+    echo "${JAVA_HOME}"
+fi
 
 PID_DIR="$(
     cd "${curdir}"

diff --git a/gensrc/script/gen_build_version.sh b/gensrc/script/gen_build_version.sh
@@ -27,7 +27,7 @@
 
 set -eo pipefail
 
-build_version_prefix="doris"
+build_version_prefix="selectdb-doris"
 build_version_major=2
 build_version_minor=1
 build_version_patch=0
@@ -63,7 +63,7 @@ if [[ -d '.git' ]]; then
     short_revision="$(git log -1 --pretty=format:"%h")"
     url="git://${hostname}"
 else
-    revision="Unknown"
+    revision="tag"
     short_revision="${revision}"
     url="file://${hostname}"
 fi
-Original file line number
+Diff line change
@@ Expand Up / @@ -122,6 +122,7 @@ fi @@
     echo "ARCH: ${ARCH}"
     ORI_OUTPUT="${ROOT}/output"
+    rm -rf "${ORI_OUTPUT}"
     FE="fe"
     BE="be"
@@ Expand Down @@