diff --git a/docker/disaggregate/Dockerfile_disaggregated_be_ubuntu b/docker/disaggregate/Dockerfile_disaggregated_be_ubuntu new file mode 100644 index 00000000..cd412270 --- /dev/null +++ b/docker/disaggregate/Dockerfile_disaggregated_be_ubuntu @@ -0,0 +1,14 @@ +FROM selectdb/base:latest + +RUN ln -s /usr/lib/jvm/jdk-17 /usr/lib/jvm/java + +ENV JAVA_HOME=/usr/lib/jvm/java + +ENV PATH=$PATH:$JAVA_HOME/bin:/opt/apache-doris/be/bin + +COPY resource/selectdb-cloud-4.0-ubuntu/be /opt/apache-doris/be + +COPY resource/be_disaggregated_* /opt/apache-doris/ + + +WORKDIR /opt/apache-doris diff --git a/docker/disaggregate/Dockerfile_disaggregated_fe_ubuntu b/docker/disaggregate/Dockerfile_disaggregated_fe_ubuntu new file mode 100644 index 00000000..e8017ed0 --- /dev/null +++ b/docker/disaggregate/Dockerfile_disaggregated_fe_ubuntu @@ -0,0 +1,14 @@ +FROM selectdb/base:latest + +RUN ln -s /usr/lib/jvm/jdk-17 /usr/lib/jvm/java + +ENV JAVA_HOME=/usr/lib/jvm/java + +ENV PATH=$PATH:$JAVA_HOME/bin:/opt/apache-doris/fe/bin + +COPY resource/selectdb-cloud-4.0-ubuntu/fe /opt/apache-doris/fe + +COPY resource/fe_disaggregated_* /opt/apache-doris + +WORKDIR /opt/apache-doris + diff --git a/docker/disaggregate/ms/DockerFile b/docker/disaggregate/Dockerfile_disaggregated_ms_ubuntu similarity index 100% rename from docker/disaggregate/ms/DockerFile rename to docker/disaggregate/Dockerfile_disaggregated_ms_ubuntu diff --git a/docker/disaggregate/resource/be_disaggregated_entrypoint.sh b/docker/disaggregate/resource/be_disaggregated_entrypoint.sh new file mode 100755 index 00000000..6224609b --- /dev/null +++ b/docker/disaggregate/resource/be_disaggregated_entrypoint.sh @@ -0,0 +1,166 @@ +#!/bin/bash +#TODO: convert to "_" +MS_ENDPOINT=${MS_ENDPOINT} +MS_TOKEN=${MS_TOKEN:="greedisgood9999"} +CLOUDE_UNIQUE_ID=${CLOUD_UNIQUE_ID} +DORIS_HOME=${DORIS_HOME:="/opt/apache-doris"} +CONFIGMAP_PATH=${CONFIGMAP_PATH:="/etc/doris"} +INSTANCE_ID=${INSTANCE_ID} +INSTANCE_NAME=${INSTANCE_NAME} +HEARTBEAT_PORT=9050 +CLUSTER_NMAE=${CLUSTER_NAME} +#option:IP,FQDN +HOST_TYPE=${HOST_TYPE:="FQDN"} +STATEFULSET_NAME=${STATEFULSET_NAME} +POD_NAMESPACE=$POD_NAMESPACE +DEFAULT_CLUSTER_ID=${POD_NAMESPACE}"_"${STATEFULSET_NAME} +CLUSTER_ID=${CLUSTER_ID:="$DEFAULT_CLUSTER_ID"} +CLOUD_UNIQUE_ID=${CLOUD_UNIQUE_ID:="1:$INSTANCE_ID:$STATEFULSET_NAME"} +# replace "-" with "_" in CLUSTER_ID and CLOUD_UNIQUE_ID +CLUSTER_ID=$(sed 's/-/_/g' <<<$CLUSTER_ID) +CLOUD_UNIQUE_ID=$(sed 's/-/_/g' <<<$CLOUD_UNIQUE_ID) +CONFIG_FILE="$DORIS_HOME/be/conf/be.conf" +MY_SELF= + +DEFAULT_CLUSTER_NAME=$(awk -F $INSTANCE_NAME"-" '{print $NF}' <<<$STATEFULSET_NAME) +CLUSTER_NAME=${CLUSTER_NAME:="$DEFAULT_CLUSTER_NAME"} + +echo 'file_cache_path = [{"path":"/opt/apache-doris/be/storage","total_size":107374182400,"query_limit":107374182400}]' >> $DORIS_HOME/be/conf/be.conf + +function log_stderr() +{ + echo "[`date`] $@" >& 1 +} + +function add_cluster_info_to_conf() +{ + echo "meta_service_endpoint=$MS_ENDPOINT" >> $DORIS_HOME/be/conf/be.conf + echo "cloud_unique_id=$CLOUD_UNIQUE_ID" >> $DORIS_HOME/be/conf/be.conf + echo "meta_service_use_load_balancer = false" >> $DORIS_HOME/be/conf/be.conf + echo "enable_file_cache = true" >> $DORIS_HOME/be/conf/be.conf +} + +function link_config_files() +{ + if [[ -d $CONFIGMAP_PATH ]]; then + for file in `ls $CONFIGMAP_PATH`; + do + if [[ -f $DORIS_HOME/be/conf/$file ]]; then + mv $DORIS_HOME/be/conf/$file $DORIS_HOME/be/conf/$file.bak + fi + done + fi + + for file in `ls $CONFIGMAP_PATH`; + do + if [[ "$file" == "be.conf" ]]; then + cp $CONFIGMAP_PATH/$file $DORIS_HOME/be/conf/$file + add_cluster_info_to_conf + continue + fi + + ln -sfT $CONFIGMAP_PATH/$file $DORIS_HOME/be/conf/$file + done +} + +function parse_config_file_with_key() +{ + local key=$1 + local value=`grep "^\s*$key\s*=" $CONFIG_FILE | sed "s|^\s$key\s*=\(.*\)\s*|\1|g"` +} + +function parse_my_self_address() +{ + local my_ip=`hostname -i | awk '{print $1}'` + local my_fqdn=`hostname -f` + if [[ $HOST_TYPE == "IP" ]]; then + MY_SELF=$my_ip + else + MY_SELF=$my_fqdn + fi +} + +function variables_initial() +{ + parse_my_self_address + local heartbeat_port=$(parse_config_file_with_key "heartbeat_service_port") + if [[ "x$heartbeat_port" != "x" ]]; then + HEARTBEAT_PORT=$heartbeat_port + fi +} + +function check_or_register_in_ms() +{ + interval=5 + start=$(date +%s) + timeout=60 + while true; + do + local find_address="http://$MS_ENDPOINT/MetaService/http/get_cluster?token=$MS_TOKEN" + local output=$(curl -s $find_address \ + -d '{"cloud_unique_id":"'$CLOUD_UNIQUE_ID'","cluster_id":"'$CLUSTER_ID'"}') + if grep -q -w "$MY_SELF" <<< $output &>/dev/null; then + log_stderr "[INFO] $MY_SELF have register in instance id $INSTANCE_ID cluser id $CLUSTER_ID!" + return + fi + + local code=$(jq -r ".code" <<< $output) + if [[ "$code" == "NOT_FOUND" ]]; then + # if grep -q -w "$CLUSTER_ID" <<< $output &>/dev/null; then + # log_stderr "[INFO] cluster id $CLUSTER_ID have exists, only register self.!" + # add_my_self + # else + log_stderr "[INFO] register cluster id $CLUSTER_ID with myself $MY_SELF into instance id $INSTANCE_ID." + add_my_self_with_cluster + # fi + else + log_stderr "[INFO] register $MY_SELF into cluster id $CLUSTER_ID!" + add_my_self + fi + + local now=$(date +%s) + let "expire=start+timeout" + if [[ $expire -le $now ]]; then + log_stderr "[ERROR] Timeout for register myself to ms, abort!" + exit 1 + fi + sleep $interval + done +} + +function add_my_self() +{ + local register_address="http://$MS_ENDPOINT/MetaService/http/add_node?token=$MS_TOKEN" + local output=$(curl -s $register_address \ + -d '{"instance_id":"'$INSTANCE_ID'", + "cluster":{"type":"COMPUTE","cluster_id":"'$CLUSTER_ID'", + "nodes":[{"cloud_unique_id":"'$CLOUD_UNIQUE_ID'","ip":"'$MY_SELF'","host":"'$MY_SELF'","heartbeat_port":'$HEARTBEAT_PORT'}]}}') + local code=$(jq -r ".code" <<< $output) + if [[ "$code" == "OK" ]]; then + log_stderr "[INFO] my_self $MY_SELF register to ms $MS_ENDPOINT instance_id $INSTANCE_ID be cluster $CLUSTER_ID success." + else + log_stderr "[ERROR] my_self $MY_SELF register ms $MS_ENDPOINT instance_id $INSTANCE_ID be cluster $CLUSTER_ID failed,err=$output!" + fi +} + +function add_my_self_with_cluster() +{ + local register_address="http://$MS_ENDPOINT/MetaService/http/add_cluster?token=$MS_TOKEN" + local output=$(curl -s $register_address \ + -d '{"instance_id":"'$INSTANCE_ID'", + "cluster":{"type":"COMPUTE","cluster_name":"'$CLUSTER_NAME'","cluster_id":"'$CLUSTER_ID'", + "nodes":[{"cloud_unique_id":"'$CLOUD_UNIQUE_ID'","ip":"'$MY_SELF'","host":"'$MY_SELF'","heartbeat_port":'$HEARTBEAT_PORT'}]}}') + local code=$(jq -r ".code" <<< $output) + if [[ "$code" == "OK" ]]; then + log_stderr "[INFO] cluster $CLUSTER_ID contains $MY_SELF register to ms $MS_ENDPOINT instance_id $INSTANCE_ID success." + else + log_stderr "[ERROR] cluster $CLUSTER_ID contains $MY_SELF register to ms $MS_ENDPOINT instance_id $INSTANCE_ID failed,err=$output!" + fi +} + +add_cluster_info_to_conf +link_config_files +variables_initial +check_or_register_in_ms + +$DORIS_HOME/be/bin/start_be.sh --console diff --git a/docker/disaggregate/resource/be_disaggregated_prestop.sh b/docker/disaggregate/resource/be_disaggregated_prestop.sh new file mode 100755 index 00000000..64a1c03a --- /dev/null +++ b/docker/disaggregate/resource/be_disaggregated_prestop.sh @@ -0,0 +1,6 @@ +#!/bin/bash +# + +DORIS_HOME=${DORIS_HOME:="/opt/apache-doris"} + +$DORIS_HOME/be/bin/stop_be.sh --grace diff --git a/docker/disaggregate/resource/fe_disaggregated_entrypoint.sh b/docker/disaggregate/resource/fe_disaggregated_entrypoint.sh new file mode 100755 index 00000000..df0171bd --- /dev/null +++ b/docker/disaggregate/resource/fe_disaggregated_entrypoint.sh @@ -0,0 +1,192 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +# ms address, fe pod's address should register in it. +MS_ENDPOINT=${MS_ENDPOINT} +MS_TOKEN=${MS_TOKEN:="greedisgood9999"} +FE_EDIT_PORT=${FE_EDIT_PORT:=9010} +# cloud_id is default. +CLUSTER_ID="RESERVED_CLUSTER_ID_FOR_SQL_SERVER" +# cloud_name is default. +CLUSTER_NAME="RESERVED_CLUSTER_NAME_FOR_SQL_SERVER" +#the instance id, pod's address should register in instance->cluster. +INSTANCE_ID=${INSTANCE_ID} +MY_SELF= +HOSTNAME=`hostname` +STATEFULSET_NAME=${STATEFULSET_NAME} +CLOUD_UNIQUE_ID=${CLOUD_UNIQUE_ID:="1:$INSTANCE_ID:$STATEFULSET_NAME"} + +CONFIGMAP_PATH=${CONFIGMAP_PATH:="/etc/doris"} +DORIS_HOME=${DORIS_HOME:="/opt/apache-doris"} +CONFIG_FILE="$DORIS_HOME/fe/conf/fe.conf" + +SEQUENCE_NUMBER=$(hostname | awk -F '-' '{print $NF}') +NODE_TYPE="FE_MASTER" + +if [[ "x$SEQUENCE_NUMBER" != "x0" ]]; then + NODE_TYPE="FE_OBSERVER" +fi + +# 1. add default config in config file or link config files. +# 2. assign global variables. +# 3. register myself. + + +function log_stderr() +{ + echo "[`date`] $@" >& 1 +} + +function add_cluster_info_to_conf() +{ + echo "meta_service_endpoint=$MS_ENDPOINT" >> $DORIS_HOME/fe/conf/fe.conf + echo "cloud_unique_id=$CLOUD_UNIQUE_ID" >> $DORIS_HOME/fe/conf/fe.conf +} + +function link_config_files() +{ + if [[ -d $CONFIGMAP_PATH ]]; then + #backup files want to replace + for file in `ls $CONFIGMAP_PATH`; + do + if [[ -f $DORIS_HOME/fe/conf/$file ]]; then + mv $DORIS_HOME/fe/conf/$file $DORIS_HOME/fe/conf/$file.bak + fi + done + + for file in `ls $CONFIGMAP_PATH`; + do + if [[ "$file" == "fe.conf" ]]; then + cp $CONFIGMAP_PATH/$file $DORIS_HOME/fe/conf/$file + add_cluster_info_to_conf + continue + fi + + ln -sfT $CONFIGMAP_PATH/$file $DORIS_HOME/fe/conf/$file + done + fi +} + +parse_config_file_with_key() +{ + local key=$1 + local value=`grep "^\s*$key\s*=" $CONFIG_FILE | sed "s|^\s*$key\s*=\(.*\)\s*$|\1|g"` + echo $value +} + +# confirm the register address, if config `enable_fqdn_mode=true` use fqdn start or use ip. +function parse_my_self_address() +{ + local value=`parse_config_file_with_key "enable_fqdn_mode"` + + local my_ip=`hostname -i | awk '{print $1}'` + local my_fqdn=`hostname -f` + if [[ $value == "true" ]]; then + MY_SELF=$my_fqdn + else + MY_SELF=$my_ip + fi +} + +function variables_inital() +{ + parse_my_self_address + local edit_port=$(parse_config_file_with_key "edit_log_port") + if [[ "x$edit_port" != "x" ]]; then + FE_EDIT_PORT=${edit_port:=$FE_EDIT_PORT} + fi + +} + +function check_or_register_in_ms() +{ + interval=5 + start=$(date +%s) + timeout=60 + while true; + do + local find_address="http://$MS_ENDPOINT/MetaService/http/get_cluster?token=$MS_TOKEN" + local output=$(curl -s $find_address \ + -d '{"cloud_unique_id": "'$CLOUD_UNIQUE_ID'", + "cluster_id": "RESERVED_CLUSTER_ID_FOR_SQL_SERVER"}') + if grep -q -w $MY_SELF <<< $output &>/dev/null ; then + log_stderr "[INFO] $MY_SELF have registerd in metaservice!" + return + fi + + local code=$(jq -r ".code" <<< $output) + if [[ "$code" == "NOT_FOUND" ]]; then + # if grep -q -w "RESERVED_CLUSTER_NAME_FOR_SQL_SERVER" <<< $output &>/dev/null; then + # log_stderr "[INFO] RESERVED_CLUSTER_NAME_FOR_SQL_SERVER fe cluster have exist, register node $MY_SELF." + # add_my_self + # else + log_stderr "[INFO] RESERVED_CLUSTER_NAME_FOR_SQL_SERVER fe cluster not exist, register fe clsuter." + add_my_self_with_cluster + # fi + else + log_stderr "[INFO] register myself $MY_SELF into fe cluster cloud_unique_id $CLOUD_UNIQUE_ID." + add_my_self + fi + + local now=$(date +%s) + let "expire=start+timeout" + if [[ $expire -le $now ]]; then + log_stderr "[ERROR] Timeout for register myself to ms, abort!" + exit 1 + fi + sleep $interval + done +} + +function add_my_self() +{ + local register_address="http://$MS_ENDPOINT/MetaService/http/add_node?token=$MS_TOKEN" + local output=$(curl -s $register_address \ + -d '{"instance_id":"'$INSTANCE_ID'", + "cluster":{"type":"SQL","cluster_name":"RESERVED_CLUSTER_NAME_FOR_SQL_SERVER","cluster_id":"RESERVED_CLUSTER_ID_FOR_SQL_SERVER", + "nodes":[{"cloud_unique_id":"'$CLOUD_UNIQUE_ID'","ip":"'$MY_SELF'","host":"'$MY_SELF'","edit_log_port":9010,"node_type":"'$NODE_TYPE'"}]}}') + local code=$(jq -r ".code" <<< $output) + if [[ "$code" == "OK" ]]; then + log_stderr "[INFO] my_self $MY_SELF register to ms $MS_ENDPOINT instance_id $INSTANCE_ID fe cluster RESERVED_CLUSTER_NAME_FOR_SQL_SERVER success!" + else + log_stderr "[ERROR] my_self register ms $MS_ENDPOINT instance_id $INSTANCE_ID fe cluster failed, response $output!" + fi +} + +function add_my_self_with_cluster() +{ + local register_address="http://$MS_ENDPOINT/MetaService/http/add_cluster?token=$MS_TOKEN" + local output=$(curl -s $register_address \ + -d '{"instance_id":"'$INSTANCE_ID'", + "cluster":{"type":"SQL","cluster_name":"RESERVED_CLUSTER_NAME_FOR_SQL_SERVER","cluster_id":"RESERVED_CLUSTER_ID_FOR_SQL_SERVER", + "nodes":[{"cloud_unique_id":"'$CLOUD_UNIQUE_ID'","ip":"'$MY_SELF'","host":"'$MY_SELF'","node_type":"'$NODE_TYPE',"edit_log_port":'$FE_EDIT_PORT'}]}}') + code=$(jq -r ".code" <<< $output) + if [[ "$code" == "OK" ]]; then + log_stderr "[INFO] fe cluster contains $MY_SELF node_type $NODE_TYPE register to ms $MS_ENDPOINT instance_id $INSTANCE_ID success." + else + log_stderr "[ERROR] fe cluster contains $MY_SELF node_type $NODE_TYPE register to ms $MS_ENDPOINT instance_id $INSTANCE_ID faied, $output!" + fi +} + +add_cluster_info_to_conf +link_config_files +variables_inital +check_or_register_in_ms +/opt/apache-doris/fe/bin/start_fe.sh --console + diff --git a/docker/disaggregate/ms/resource/ms_disaggregated_entrypoint.sh b/docker/disaggregate/resource/ms_disaggregated_entrypoint.sh old mode 100644 new mode 100755 similarity index 100% rename from docker/disaggregate/ms/resource/ms_disaggregated_entrypoint.sh rename to docker/disaggregate/resource/ms_disaggregated_entrypoint.sh diff --git a/docker/disaggregate/ms/resource/ms_disaggregated_is_alive.sh b/docker/disaggregate/resource/ms_disaggregated_is_alive.sh old mode 100644 new mode 100755 similarity index 100% rename from docker/disaggregate/ms/resource/ms_disaggregated_is_alive.sh rename to docker/disaggregate/resource/ms_disaggregated_is_alive.sh diff --git a/docker/disaggregate/ms/resource/ms_disaggregated_prestop.sh b/docker/disaggregate/resource/ms_disaggregated_prestop.sh old mode 100644 new mode 100755 similarity index 100% rename from docker/disaggregate/ms/resource/ms_disaggregated_prestop.sh rename to docker/disaggregate/resource/ms_disaggregated_prestop.sh