From ada43139a354fdd4e4718106b5e5bb3c71273839 Mon Sep 17 00:00:00 2001 From: yagagagaga Date: Tue, 24 Dec 2024 20:38:22 +0800 Subject: [PATCH 01/11] [chore](bash) optimize doris_cloud bootstrap information --- cloud/script/start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/script/start.sh b/cloud/script/start.sh index ecb5a3b2bed1e6..a56b1dc2bdf7f0 100644 --- a/cloud/script/start.sh +++ b/cloud/script/start.sh @@ -141,7 +141,7 @@ if [[ "${RUN_DAEMON}" -eq 1 ]]; then echo "${process} may not start successfully please check process log for more details" exit 1 fi - echo "${process} start successfully" + tail -n10 "${out_file}" | grep -E '(recycler|meta-service) started' exit 0 elif [[ "${RUN_CONSOLE}" -eq 1 ]]; then export DORIS_LOG_TO_STDERR=1 From 3b15ec1bb18d80606748d98b67a482483c8c9225 Mon Sep 17 00:00:00 2001 From: yagagagaga Date: Wed, 15 Jan 2025 17:44:44 +0800 Subject: [PATCH 02/11] just modify bash script --- cloud/script/start.sh | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/cloud/script/start.sh b/cloud/script/start.sh index a56b1dc2bdf7f0..41b9194b43d08a 100644 --- a/cloud/script/start.sh +++ b/cloud/script/start.sh @@ -35,6 +35,8 @@ fi RUN_DAEMON=0 RUN_VERSION=0 RUN_CONSOLE=0 +RUN_METASERVICE=0 +RUN_RECYCLYER=0 for arg; do shift [[ "${arg}" = "--daemonized" ]] && RUN_DAEMON=1 && continue @@ -42,6 +44,8 @@ for arg; do [[ "${arg}" = "--daemon" ]] && RUN_DAEMON=1 && continue [[ "${arg}" = "--version" ]] && RUN_VERSION=1 && continue [[ "${arg}" = "--console" ]] && RUN_CONSOLE=1 && continue + [[ "${arg}" = "--meta-service" ]] && RUN_METASERVICE=1 && continue + [[ "${arg}" = "--recycler" ]] && RUN_RECYCLYER=1 && continue set -- "$@" "${arg}" done # echo "$@" "daemonized=${daemonized}"} @@ -60,13 +64,23 @@ while read -r line; do fi done <"${DORIS_HOME}/conf/doris_cloud.conf" +role='' +if [[ ${RUN_METASERVICE} -eq 0 ]] && [[ ${RUN_RECYCLYER} -eq 0 ]]; then + role='MetaService and Recycler' +elif [[ ${RUN_METASERVICE} -eq 1 ]] && [[ ${RUN_RECYCLYER} -eq 0 ]]; then + role='MetaService' +elif [[ ${RUN_METASERVICE} -eq 0 ]] && [[ ${RUN_RECYCLYER} -eq 1 ]]; then + role='Recycler' +elif [[ ${RUN_METASERVICE} -eq 1 ]] && [[ ${RUN_RECYCLYER} -eq 1 ]]; then + role='MetaService and Recycler' +fi process=doris_cloud if [[ ${RUN_VERSION} -eq 0 ]] && [[ -f "${DORIS_HOME}/bin/${process}.pid" ]]; then pid=$(cat "${DORIS_HOME}/bin/${process}.pid") if [[ "${pid}" != "" ]]; then if kill -0 "$(cat "${DORIS_HOME}/bin/${process}.pid")" >/dev/null 2>&1; then - echo "pid file existed, ${process} have already started, pid=${pid}" + echo "pid file existed, ${role} have already started, pid=${pid}" exit 1 fi fi @@ -113,7 +127,7 @@ if [[ -f "${DORIS_HOME}/conf/hdfs-site.xml" ]]; then export LIBHDFS3_CONF="${DORIS_HOME}/conf/hdfs-site.xml" fi -echo "LIBHDFS3_CONF=${LIBHDFS3_CONF}" +# echo "LIBHDFS3_CONF=${LIBHDFS3_CONF}" # to enable dump jeprof heap stats prodigally, change `prof_active:false` to `prof_active:true` or curl http://be_host:be_webport/jeheap/prof/true # to control the dump interval change `lg_prof_interval` to a specific value, it is pow/exponent of 2 in size of bytes, default 34 means 2 ** 34 = 16GB @@ -126,22 +140,19 @@ if [[ "${RUN_VERSION}" -ne 0 ]]; then fi mkdir -p "${DORIS_HOME}/log" -echo "starts ${process} with args: $*" out_file=${DORIS_HOME}/log/${process}.out +# append 10 blank lines to ensure the following tail -n10 works correctly +printf "\n\n\n\n\n\n\n\n\n\n" >>"${out_file}" +echo "$(date +'%F %T') start with args: $*" >>"${out_file}" if [[ "${RUN_DAEMON}" -eq 1 ]]; then - # append 10 blank lines to ensure the following tail -n10 works correctly - printf "\n\n\n\n\n\n\n\n\n\n" >>"${out_file}" - echo "$(date +'%F %T') try to start ${process}" >>"${out_file}" nohup "${bin}" "$@" >>"${out_file}" 2>&1 & - echo "wait and check ${process} start successfully" + echo "wait and check ${role} start successfully" >>"${out_file}" sleep 3 - tail -n10 "${out_file}" | grep 'successfully started brpc' - ret=$? if [[ ${ret} -ne 0 ]]; then - echo "${process} may not start successfully please check process log for more details" + echo "${role} may not start successfully please check process log for more details" exit 1 fi - tail -n10 "${out_file}" | grep -E '(recycler|meta-service) started' + tail -n12 "${out_file}" exit 0 elif [[ "${RUN_CONSOLE}" -eq 1 ]]; then export DORIS_LOG_TO_STDERR=1 From a2035a49a7ff35ec08d356f3f5ca987fa43253bc Mon Sep 17 00:00:00 2001 From: yagagagaga Date: Wed, 15 Jan 2025 17:53:24 +0800 Subject: [PATCH 03/11] add a empty line --- cloud/src/main.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cloud/src/main.cpp b/cloud/src/main.cpp index 2ea251a2d2abf0..fb9b36fbaa5f84 100644 --- a/cloud/src/main.cpp +++ b/cloud/src/main.cpp @@ -261,6 +261,7 @@ int main(int argc, char** argv) { LOG(INFO) << "successfully init txn kv, elapsed milliseconds: " << duration_cast(end - start_init_kv).count(); + if (init_global_encryption_key_info_map(txn_kv.get()) != 0) { LOG(WARNING) << "failed to init global encryption key map"; return -1; From d082f9f494c28f0cd7bbf40a57311c7b3fef7c5d Mon Sep 17 00:00:00 2001 From: yagagagaga Date: Wed, 15 Jan 2025 19:15:07 +0800 Subject: [PATCH 04/11] roll back script --- cloud/script/start.sh | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/cloud/script/start.sh b/cloud/script/start.sh index 41b9194b43d08a..ecb5a3b2bed1e6 100644 --- a/cloud/script/start.sh +++ b/cloud/script/start.sh @@ -35,8 +35,6 @@ fi RUN_DAEMON=0 RUN_VERSION=0 RUN_CONSOLE=0 -RUN_METASERVICE=0 -RUN_RECYCLYER=0 for arg; do shift [[ "${arg}" = "--daemonized" ]] && RUN_DAEMON=1 && continue @@ -44,8 +42,6 @@ for arg; do [[ "${arg}" = "--daemon" ]] && RUN_DAEMON=1 && continue [[ "${arg}" = "--version" ]] && RUN_VERSION=1 && continue [[ "${arg}" = "--console" ]] && RUN_CONSOLE=1 && continue - [[ "${arg}" = "--meta-service" ]] && RUN_METASERVICE=1 && continue - [[ "${arg}" = "--recycler" ]] && RUN_RECYCLYER=1 && continue set -- "$@" "${arg}" done # echo "$@" "daemonized=${daemonized}"} @@ -64,23 +60,13 @@ while read -r line; do fi done <"${DORIS_HOME}/conf/doris_cloud.conf" -role='' -if [[ ${RUN_METASERVICE} -eq 0 ]] && [[ ${RUN_RECYCLYER} -eq 0 ]]; then - role='MetaService and Recycler' -elif [[ ${RUN_METASERVICE} -eq 1 ]] && [[ ${RUN_RECYCLYER} -eq 0 ]]; then - role='MetaService' -elif [[ ${RUN_METASERVICE} -eq 0 ]] && [[ ${RUN_RECYCLYER} -eq 1 ]]; then - role='Recycler' -elif [[ ${RUN_METASERVICE} -eq 1 ]] && [[ ${RUN_RECYCLYER} -eq 1 ]]; then - role='MetaService and Recycler' -fi process=doris_cloud if [[ ${RUN_VERSION} -eq 0 ]] && [[ -f "${DORIS_HOME}/bin/${process}.pid" ]]; then pid=$(cat "${DORIS_HOME}/bin/${process}.pid") if [[ "${pid}" != "" ]]; then if kill -0 "$(cat "${DORIS_HOME}/bin/${process}.pid")" >/dev/null 2>&1; then - echo "pid file existed, ${role} have already started, pid=${pid}" + echo "pid file existed, ${process} have already started, pid=${pid}" exit 1 fi fi @@ -127,7 +113,7 @@ if [[ -f "${DORIS_HOME}/conf/hdfs-site.xml" ]]; then export LIBHDFS3_CONF="${DORIS_HOME}/conf/hdfs-site.xml" fi -# echo "LIBHDFS3_CONF=${LIBHDFS3_CONF}" +echo "LIBHDFS3_CONF=${LIBHDFS3_CONF}" # to enable dump jeprof heap stats prodigally, change `prof_active:false` to `prof_active:true` or curl http://be_host:be_webport/jeheap/prof/true # to control the dump interval change `lg_prof_interval` to a specific value, it is pow/exponent of 2 in size of bytes, default 34 means 2 ** 34 = 16GB @@ -140,19 +126,22 @@ if [[ "${RUN_VERSION}" -ne 0 ]]; then fi mkdir -p "${DORIS_HOME}/log" +echo "starts ${process} with args: $*" out_file=${DORIS_HOME}/log/${process}.out -# append 10 blank lines to ensure the following tail -n10 works correctly -printf "\n\n\n\n\n\n\n\n\n\n" >>"${out_file}" -echo "$(date +'%F %T') start with args: $*" >>"${out_file}" if [[ "${RUN_DAEMON}" -eq 1 ]]; then + # append 10 blank lines to ensure the following tail -n10 works correctly + printf "\n\n\n\n\n\n\n\n\n\n" >>"${out_file}" + echo "$(date +'%F %T') try to start ${process}" >>"${out_file}" nohup "${bin}" "$@" >>"${out_file}" 2>&1 & - echo "wait and check ${role} start successfully" >>"${out_file}" + echo "wait and check ${process} start successfully" sleep 3 + tail -n10 "${out_file}" | grep 'successfully started brpc' + ret=$? if [[ ${ret} -ne 0 ]]; then - echo "${role} may not start successfully please check process log for more details" + echo "${process} may not start successfully please check process log for more details" exit 1 fi - tail -n12 "${out_file}" + echo "${process} start successfully" exit 0 elif [[ "${RUN_CONSOLE}" -eq 1 ]]; then export DORIS_LOG_TO_STDERR=1 From c17200df1ce13d98be18aac4852cc7e661431e0a Mon Sep 17 00:00:00 2001 From: yagagagaga Date: Wed, 15 Jan 2025 21:01:15 +0800 Subject: [PATCH 05/11] just update some string --- cloud/src/main.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cloud/src/main.cpp b/cloud/src/main.cpp index fb9b36fbaa5f84..e0ed46e1684c4a 100644 --- a/cloud/src/main.cpp +++ b/cloud/src/main.cpp @@ -230,7 +230,7 @@ int main(int argc, char** argv) { std::get<0>(args.args()[ARG_RECYCLER]) = true; LOG(INFO) << "meta_service and recycler are both not specified, " "run doris_cloud as meta_service and recycler by default"; - std::cout << "run doris_cloud as meta_service and recycler by default" << std::endl; + std::cout << "try to start meta_service, recycler" << std::endl; } brpc::Server server; @@ -283,7 +283,7 @@ int main(int argc, char** argv) { std::cerr << msg << std::endl; return ret; } - msg = "meta-service started"; + msg = "MetaService has been started successfully"; LOG(INFO) << msg; std::cout << msg << std::endl; } @@ -296,7 +296,7 @@ int main(int argc, char** argv) { std::cerr << msg << std::endl; return ret; } - msg = "recycler started"; + msg = "Recycler has been started successfully"; LOG(INFO) << msg; std::cout << msg << std::endl; auto periodiccally_log = [&]() { From 289265c44beb0c5f3ffbb173f188bb879237c807 Mon Sep 17 00:00:00 2001 From: yagagagaga Date: Wed, 15 Jan 2025 21:02:01 +0800 Subject: [PATCH 06/11] update --- cloud/src/main.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/cloud/src/main.cpp b/cloud/src/main.cpp index e0ed46e1684c4a..987e26df705057 100644 --- a/cloud/src/main.cpp +++ b/cloud/src/main.cpp @@ -261,7 +261,6 @@ int main(int argc, char** argv) { LOG(INFO) << "successfully init txn kv, elapsed milliseconds: " << duration_cast(end - start_init_kv).count(); - if (init_global_encryption_key_info_map(txn_kv.get()) != 0) { LOG(WARNING) << "failed to init global encryption key map"; return -1; From ee0c9190e35f0c7af7175d98442c9df9fe50f1b5 Mon Sep 17 00:00:00 2001 From: yagagagaga Date: Thu, 16 Jan 2025 11:36:39 +0800 Subject: [PATCH 07/11] update script --- cloud/script/start.sh | 2 +- cloud/src/main.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cloud/script/start.sh b/cloud/script/start.sh index ecb5a3b2bed1e6..dd9163283e5ba0 100644 --- a/cloud/script/start.sh +++ b/cloud/script/start.sh @@ -135,7 +135,7 @@ if [[ "${RUN_DAEMON}" -eq 1 ]]; then nohup "${bin}" "$@" >>"${out_file}" 2>&1 & echo "wait and check ${process} start successfully" sleep 3 - tail -n10 "${out_file}" | grep 'successfully started brpc' + tail -n10 "${out_file}" | grep 'successfully started service' ret=$? if [[ ${ret} -ne 0 ]]; then echo "${process} may not start successfully please check process log for more details" diff --git a/cloud/src/main.cpp b/cloud/src/main.cpp index 987e26df705057..32667d0e1a2478 100644 --- a/cloud/src/main.cpp +++ b/cloud/src/main.cpp @@ -221,7 +221,7 @@ int main(int argc, char** argv) { // We can invoke glog from now on std::string msg; - LOG(INFO) << "try to start doris_cloud"; + LOG(INFO) << "try to start " << process_name; LOG(INFO) << build_info(); std::cout << build_info() << std::endl; @@ -324,7 +324,7 @@ int main(int argc, char** argv) { return -1; } end = steady_clock::now(); - msg = "successfully started brpc listening on port=" + std::to_string(port) + + msg = "successfully started service listening on port=" + std::to_string(port) + " time_elapsed_ms=" + std::to_string(duration_cast(end - start).count()); LOG(INFO) << msg; std::cout << msg << std::endl; From d141fa8e86fb9e740dc3f424efab59fb50b3805b Mon Sep 17 00:00:00 2001 From: yagagagaga Date: Thu, 16 Jan 2025 13:02:19 +0800 Subject: [PATCH 08/11] update script --- cloud/script/start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/script/start.sh b/cloud/script/start.sh index dd9163283e5ba0..e121fac8781c49 100644 --- a/cloud/script/start.sh +++ b/cloud/script/start.sh @@ -141,7 +141,7 @@ if [[ "${RUN_DAEMON}" -eq 1 ]]; then echo "${process} may not start successfully please check process log for more details" exit 1 fi - echo "${process} start successfully" + tail -n12 "${out_file}" exit 0 elif [[ "${RUN_CONSOLE}" -eq 1 ]]; then export DORIS_LOG_TO_STDERR=1 From 9c913ac59cf5e4b33d43b7683e2cd03231465cb8 Mon Sep 17 00:00:00 2001 From: yagagagaga Date: Thu, 16 Jan 2025 15:22:47 +0800 Subject: [PATCH 09/11] update script --- cloud/script/start.sh | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/cloud/script/start.sh b/cloud/script/start.sh index e121fac8781c49..909fd51b74d932 100644 --- a/cloud/script/start.sh +++ b/cloud/script/start.sh @@ -35,6 +35,8 @@ fi RUN_DAEMON=0 RUN_VERSION=0 RUN_CONSOLE=0 +RUN_METASERVICE=0 +RUN_RECYCLYER=0 for arg; do shift [[ "${arg}" = "--daemonized" ]] && RUN_DAEMON=1 && continue @@ -42,6 +44,8 @@ for arg; do [[ "${arg}" = "--daemon" ]] && RUN_DAEMON=1 && continue [[ "${arg}" = "--version" ]] && RUN_VERSION=1 && continue [[ "${arg}" = "--console" ]] && RUN_CONSOLE=1 && continue + [[ "${arg}" = "--meta-service" ]] && RUN_METASERVICE=1 && continue + [[ "${arg}" = "--recycler" ]] && RUN_RECYCLYER=1 && continue set -- "$@" "${arg}" done # echo "$@" "daemonized=${daemonized}"} @@ -60,13 +64,23 @@ while read -r line; do fi done <"${DORIS_HOME}/conf/doris_cloud.conf" +role='' +if [[ ${RUN_METASERVICE} -eq 0 ]] && [[ ${RUN_RECYCLYER} -eq 0 ]]; then + role='MetaService and Recycler' +elif [[ ${RUN_METASERVICE} -eq 1 ]] && [[ ${RUN_RECYCLYER} -eq 0 ]]; then + role='MetaService' +elif [[ ${RUN_METASERVICE} -eq 0 ]] && [[ ${RUN_RECYCLYER} -eq 1 ]]; then + role='Recycler' +elif [[ ${RUN_METASERVICE} -eq 1 ]] && [[ ${RUN_RECYCLYER} -eq 1 ]]; then + role='MetaService and Recycler' +fi process=doris_cloud if [[ ${RUN_VERSION} -eq 0 ]] && [[ -f "${DORIS_HOME}/bin/${process}.pid" ]]; then pid=$(cat "${DORIS_HOME}/bin/${process}.pid") if [[ "${pid}" != "" ]]; then if kill -0 "$(cat "${DORIS_HOME}/bin/${process}.pid")" >/dev/null 2>&1; then - echo "pid file existed, ${process} have already started, pid=${pid}" + echo "pid file existed, ${role} have already started, pid=${pid}" exit 1 fi fi @@ -113,7 +127,7 @@ if [[ -f "${DORIS_HOME}/conf/hdfs-site.xml" ]]; then export LIBHDFS3_CONF="${DORIS_HOME}/conf/hdfs-site.xml" fi -echo "LIBHDFS3_CONF=${LIBHDFS3_CONF}" +# echo "LIBHDFS3_CONF=${LIBHDFS3_CONF}" # to enable dump jeprof heap stats prodigally, change `prof_active:false` to `prof_active:true` or curl http://be_host:be_webport/jeheap/prof/true # to control the dump interval change `lg_prof_interval` to a specific value, it is pow/exponent of 2 in size of bytes, default 34 means 2 ** 34 = 16GB @@ -126,19 +140,18 @@ if [[ "${RUN_VERSION}" -ne 0 ]]; then fi mkdir -p "${DORIS_HOME}/log" -echo "starts ${process} with args: $*" out_file=${DORIS_HOME}/log/${process}.out if [[ "${RUN_DAEMON}" -eq 1 ]]; then # append 10 blank lines to ensure the following tail -n10 works correctly printf "\n\n\n\n\n\n\n\n\n\n" >>"${out_file}" - echo "$(date +'%F %T') try to start ${process}" >>"${out_file}" + echo "$(date +'%F %T') start with args: $*" >>"${out_file}" nohup "${bin}" "$@" >>"${out_file}" 2>&1 & - echo "wait and check ${process} start successfully" + echo "wait and check ${role} start successfully" >>"${out_file}" sleep 3 - tail -n10 "${out_file}" | grep 'successfully started service' + tail -n12 "${out_file}" | grep 'successfully started service' ret=$? if [[ ${ret} -ne 0 ]]; then - echo "${process} may not start successfully please check process log for more details" + echo "${role} may not start successfully please check process log for more details" exit 1 fi tail -n12 "${out_file}" From fd44740fadabebfedb5da201d7ce427a5dd47c44 Mon Sep 17 00:00:00 2001 From: yagagagaga Date: Thu, 16 Jan 2025 15:25:23 +0800 Subject: [PATCH 10/11] update script --- cloud/script/start.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud/script/start.sh b/cloud/script/start.sh index 909fd51b74d932..c56c2f27c24f56 100644 --- a/cloud/script/start.sh +++ b/cloud/script/start.sh @@ -140,6 +140,7 @@ if [[ "${RUN_VERSION}" -ne 0 ]]; then fi mkdir -p "${DORIS_HOME}/log" +echo "$(date +'%F %T') start with args: $*" out_file=${DORIS_HOME}/log/${process}.out if [[ "${RUN_DAEMON}" -eq 1 ]]; then # append 10 blank lines to ensure the following tail -n10 works correctly @@ -158,7 +159,6 @@ if [[ "${RUN_DAEMON}" -eq 1 ]]; then exit 0 elif [[ "${RUN_CONSOLE}" -eq 1 ]]; then export DORIS_LOG_TO_STDERR=1 - date "${bin}" "$@" 2>&1 else "${bin}" "$@" From 4c8a81c67ad88b53ec37beaf75b5b5912b5ca7ad Mon Sep 17 00:00:00 2001 From: yagagagaga Date: Thu, 16 Jan 2025 19:20:04 +0800 Subject: [PATCH 11/11] update script --- cloud/script/start.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cloud/script/start.sh b/cloud/script/start.sh index c56c2f27c24f56..9e906d5162d450 100644 --- a/cloud/script/start.sh +++ b/cloud/script/start.sh @@ -48,6 +48,12 @@ for arg; do [[ "${arg}" = "--recycler" ]] && RUN_RECYCLYER=1 && continue set -- "$@" "${arg}" done +if [[ ${RUN_METASERVICE} -eq 1 ]]; then + set -- "$@" "--meta-service" +fi +if [[ ${RUN_RECYCLYER} -eq 1 ]]; then + set -- "$@" "--recycler" +fi # echo "$@" "daemonized=${daemonized}"} # export env variables from doris_cloud.conf