From f7bff94ecab2e99a5fc062db14248f4fe336d6ff Mon Sep 17 00:00:00 2001 From: yiguolei Date: Wed, 30 Jul 2025 20:08:15 +0800 Subject: [PATCH 01/10] add glog sink --- be/src/service/doris_main.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp index 08fdeaa7c351be..d0330dc168c4af 100644 --- a/be/src/service/doris_main.cpp +++ b/be/src/service/doris_main.cpp @@ -310,6 +310,20 @@ struct Checker { #endif ; +// This method is in logging.cc in glog. +extern void ColoredWriteToStdout(LogSeverity severity, const char* message, size_t len); +// GLog has flags FLAGS_logtostderr, FLAGS_logtostdout, FLAGS_alsologtostderr. But not +// has flags like FLAGS_alsologtostdout. We need to log BE.INFO into stdout for k8s usage. +// Because log stack trace need to use stderr to output the log to be.out. +// And be.info need stdout to allow kubernetes to collect logs. +struct StdoutLogSink : google::LogSink { + void send(google::LogSeverity severity, const char* /*full_filename*/, + const char* base_filename, int line, const google::LogMessageTime& /*time*/, + const char* message, std::size_t message_len) override { + ColoredWriteToStdout(severity, message, message_len); + } +}; + int main(int argc, char** argv) { doris::signal::InstallFailureSignalHandler(); // create StackTraceCache Instance, at the beginning, other static destructors may use. @@ -387,6 +401,9 @@ int main(int argc, char** argv) { // ATTN: MUST init before LOG doris::init_glog("be"); + StdoutLogSink sink; + google::AddLogSink(&sink); + LOG(INFO) << doris::get_version_string(false); doris::init_thrift_logging(); From f2762ab34a72a3c04c5f5d9b537cae22965f9386 Mon Sep 17 00:00:00 2001 From: yiguolei Date: Thu, 31 Jul 2025 09:18:27 +0800 Subject: [PATCH 02/10] f --- be/src/common/logconfig.cpp | 9 --------- be/src/service/doris_main.cpp | 13 +++++++++++-- bin/start_be.sh | 4 +++- 3 files changed, 14 insertions(+), 12 deletions(-) diff --git a/be/src/common/logconfig.cpp b/be/src/common/logconfig.cpp index 88c7d4bc75530c..82e7a73f169048 100644 --- a/be/src/common/logconfig.cpp +++ b/be/src/common/logconfig.cpp @@ -97,15 +97,6 @@ bool init_glog(const char* basename) { return true; } - bool log_to_console = (getenv("DORIS_LOG_TO_STDERR") != nullptr); - if (log_to_console) { - if (config::enable_file_logger) { - FLAGS_alsologtostderr = true; - } else { - FLAGS_logtostderr = true; - } - } - // don't log to stderr except fatal level // so fatal log can output to be.out . FLAGS_stderrthreshold = google::FATAL; diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp index d0330dc168c4af..04bd93cda93c3f 100644 --- a/be/src/service/doris_main.cpp +++ b/be/src/service/doris_main.cpp @@ -400,9 +400,18 @@ int main(int argc, char** argv) { google::ParseCommandLineFlags(&argc, &argv, true); // ATTN: MUST init before LOG doris::init_glog("be"); - + bool log_to_console = (getenv("DORIS_LOG_TO_STDERR") != nullptr); StdoutLogSink sink; - google::AddLogSink(&sink); + if (log_to_console) { + if (config::enable_file_logger) { + // will output log to be.info and output log to stdout + google::AddLogSink(&sink); + } else { + // enable_file_logger is false, will only output log to stdout + // Not output to stderr because be.out will output log to stderr + FLAGS_logtostdout = true; + } + } LOG(INFO) << doris::get_version_string(false); diff --git a/bin/start_be.sh b/bin/start_be.sh index 04cac90818bae8..3cd88d11524ed1 100755 --- a/bin/start_be.sh +++ b/bin/start_be.sh @@ -459,8 +459,10 @@ elif [[ "${RUN_DAEMON}" -eq 1 ]]; then nohup ${LIMIT:+${LIMIT}} "${DORIS_HOME}/lib/doris_be" "$@" >>"${LOG_DIR}/be.out" 2>&1 &1 >"${LOG_DIR}/be.out" >"${LOG_DIR}/be.out" 2>&1 Date: Thu, 31 Jul 2025 09:19:17 +0800 Subject: [PATCH 03/10] f --- be/src/service/doris_main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp index 04bd93cda93c3f..a0b63f1c725e88 100644 --- a/be/src/service/doris_main.cpp +++ b/be/src/service/doris_main.cpp @@ -311,7 +311,7 @@ struct Checker { ; // This method is in logging.cc in glog. -extern void ColoredWriteToStdout(LogSeverity severity, const char* message, size_t len); +extern void ColoredWriteToStdout(google::LogSeverity severity, const char* message, size_t len); // GLog has flags FLAGS_logtostderr, FLAGS_logtostdout, FLAGS_alsologtostderr. But not // has flags like FLAGS_alsologtostdout. We need to log BE.INFO into stdout for k8s usage. // Because log stack trace need to use stderr to output the log to be.out. From 0d7aa247b1f0a999baf70044fbfd6f47f93d63c1 Mon Sep 17 00:00:00 2001 From: yiguolei Date: Thu, 31 Jul 2025 09:21:58 +0800 Subject: [PATCH 04/10] f --- be/src/common/logconfig.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/be/src/common/logconfig.cpp b/be/src/common/logconfig.cpp index 82e7a73f169048..c70546d45a69d4 100644 --- a/be/src/common/logconfig.cpp +++ b/be/src/common/logconfig.cpp @@ -97,6 +97,7 @@ bool init_glog(const char* basename) { return true; } + bool log_to_console = (getenv("DORIS_LOG_TO_STDERR") != nullptr); // don't log to stderr except fatal level // so fatal log can output to be.out . FLAGS_stderrthreshold = google::FATAL; From adbe2b0d31bf8c9c179cfb77680f9ce634d25f36 Mon Sep 17 00:00:00 2001 From: yiguolei Date: Thu, 31 Jul 2025 09:23:20 +0800 Subject: [PATCH 05/10] f --- be/src/service/doris_main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp index a0b63f1c725e88..6d084c29c0d4b9 100644 --- a/be/src/service/doris_main.cpp +++ b/be/src/service/doris_main.cpp @@ -403,7 +403,7 @@ int main(int argc, char** argv) { bool log_to_console = (getenv("DORIS_LOG_TO_STDERR") != nullptr); StdoutLogSink sink; if (log_to_console) { - if (config::enable_file_logger) { + if (doris::config::enable_file_logger) { // will output log to be.info and output log to stdout google::AddLogSink(&sink); } else { From 888832118e5ea5fff8b99fc68c95380d1a68cae5 Mon Sep 17 00:00:00 2001 From: yiguolei Date: Thu, 31 Jul 2025 09:45:04 +0800 Subject: [PATCH 06/10] f --- be/src/common/logconfig.cpp | 66 +++++++++++++++++++++++++++++++++++ be/src/service/doris_main.cpp | 26 -------------- 2 files changed, 66 insertions(+), 26 deletions(-) diff --git a/be/src/common/logconfig.cpp b/be/src/common/logconfig.cpp index c70546d45a69d4..55859044f3ea22 100644 --- a/be/src/common/logconfig.cpp +++ b/be/src/common/logconfig.cpp @@ -36,6 +36,60 @@ static bool logging_initialized = false; static std::mutex logging_mutex; +static StdoutLogSink stdout_log_sink; + +// Implement the custom log format: I20250118 10:53:06.239614 1318521 timezone_utils.cpp:115] Preloaded653 timezones. +struct StdoutLogSink : google::LogSink { + void send(google::LogSeverity severity, const char* /*full_filename*/, + const char* base_filename, int line, const google::LogMessageTime& time, + const char* message, std::size_t message_len) override { + // 1. Convert log severity to corresponding character (I/W/E/F) + char severity_char; + switch (severity) { + case google::GLOG_INFO: + severity_char = 'I'; + break; + case google::GLOG_WARNING: + severity_char = 'W'; + break; + case google::GLOG_ERROR: + severity_char = 'E'; + break; + case google::GLOG_FATAL: + severity_char = 'F'; + break; + default: + severity_char = '?'; + break; + } + + // 2. Get timestamp components + time_t t = time.time(); + struct tm tm; + localtime_r(&t, &tm); // Thread-safe local time conversion + + // Extract microseconds from the timestamp + int microseconds = static_cast((time.usec() % 1000000)); + + // 3. Get process ID + pid_t pid = getpid(); + + // 格式：级别 + 日期 + 时间 + 进程ID + 文件名:行号] 消息 + std::cout << severity_char << std::setfill('0') << std::setw(4) << tm.tm_year + 1900 // 年份 + << std::setw(2) << tm.tm_mon + 1 // 月份 + << std::setw(2) << tm.tm_mday // 日期 + << " " << std::setw(2) << tm.tm_hour // 小时 + << ":" << std::setw(2) << tm.tm_min // 分钟 + << ":" << std::setw(2) << tm.tm_sec // 秒 + << "." << std::setw(6) << microseconds // 微秒 + << " " << pid // 进程ID + << " " << base_filename // 文件名 + << ":" << line // 行号 + << "] " << std::string(message, message_len) // 日志消息 + << std::endl; + } +}; + static bool iequals(const std::string& a, const std::string& b) { unsigned int sz = a.size(); if (b.size() != sz) { @@ -98,6 +152,17 @@ bool init_glog(const char* basename) { } bool log_to_console = (getenv("DORIS_LOG_TO_STDERR") != nullptr); + if (log_to_console) { + if (doris::config::enable_file_logger) { + // will output log to be.info and output log to stdout + google::AddLogSink(&stdout_log_sink); + } else { + // enable_file_logger is false, will only output log to stdout + // Not output to stderr because be.out will output log to stderr + FLAGS_logtostdout = true; + } + } + // don't log to stderr except fatal level // so fatal log can output to be.out . FLAGS_stderrthreshold = google::FATAL; @@ -205,6 +270,7 @@ bool init_glog(const char* basename) { void shutdown_logging() { std::lock_guard logging_lock(logging_mutex); + google::RemoveLogSink(&stdcout_sink); google::ShutdownGoogleLogging(); } diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp index 6d084c29c0d4b9..08fdeaa7c351be 100644 --- a/be/src/service/doris_main.cpp +++ b/be/src/service/doris_main.cpp @@ -310,20 +310,6 @@ struct Checker { #endif ; -// This method is in logging.cc in glog. -extern void ColoredWriteToStdout(google::LogSeverity severity, const char* message, size_t len); -// GLog has flags FLAGS_logtostderr, FLAGS_logtostdout, FLAGS_alsologtostderr. But not -// has flags like FLAGS_alsologtostdout. We need to log BE.INFO into stdout for k8s usage. -// Because log stack trace need to use stderr to output the log to be.out. -// And be.info need stdout to allow kubernetes to collect logs. -struct StdoutLogSink : google::LogSink { - void send(google::LogSeverity severity, const char* /*full_filename*/, - const char* base_filename, int line, const google::LogMessageTime& /*time*/, - const char* message, std::size_t message_len) override { - ColoredWriteToStdout(severity, message, message_len); - } -}; - int main(int argc, char** argv) { doris::signal::InstallFailureSignalHandler(); // create StackTraceCache Instance, at the beginning, other static destructors may use. @@ -400,18 +386,6 @@ int main(int argc, char** argv) { google::ParseCommandLineFlags(&argc, &argv, true); // ATTN: MUST init before LOG doris::init_glog("be"); - bool log_to_console = (getenv("DORIS_LOG_TO_STDERR") != nullptr); - StdoutLogSink sink; - if (log_to_console) { - if (doris::config::enable_file_logger) { - // will output log to be.info and output log to stdout - google::AddLogSink(&sink); - } else { - // enable_file_logger is false, will only output log to stdout - // Not output to stderr because be.out will output log to stderr - FLAGS_logtostdout = true; - } - } LOG(INFO) << doris::get_version_string(false); From f5af4cdc0f10f5839a6cce79cc21a4696e163332 Mon Sep 17 00:00:00 2001 From: yiguolei Date: Thu, 31 Jul 2025 09:47:51 +0800 Subject: [PATCH 07/10] f --- be/src/common/logconfig.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/be/src/common/logconfig.cpp b/be/src/common/logconfig.cpp index 55859044f3ea22..0d44cf16918b9e 100644 --- a/be/src/common/logconfig.cpp +++ b/be/src/common/logconfig.cpp @@ -36,8 +36,6 @@ static bool logging_initialized = false; static std::mutex logging_mutex; -static StdoutLogSink stdout_log_sink; - // Implement the custom log format: I20250118 10:53:06.239614 1318521 timezone_utils.cpp:115] Preloaded653 timezones. struct StdoutLogSink : google::LogSink { void send(google::LogSeverity severity, const char* /*full_filename*/, @@ -90,6 +88,8 @@ struct StdoutLogSink : google::LogSink { } }; +static StdoutLogSink stdout_log_sink; + static bool iequals(const std::string& a, const std::string& b) { unsigned int sz = a.size(); if (b.size() != sz) { From a888c5c860fd95826cb69cb0525aa3eb44db1247 Mon Sep 17 00:00:00 2001 From: yiguolei Date: Thu, 31 Jul 2025 10:02:23 +0800 Subject: [PATCH 08/10] f --- be/src/common/logconfig.cpp | 50 +++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/be/src/common/logconfig.cpp b/be/src/common/logconfig.cpp index 0d44cf16918b9e..fcec2a0c92f732 100644 --- a/be/src/common/logconfig.cpp +++ b/be/src/common/logconfig.cpp @@ -60,31 +60,33 @@ struct StdoutLogSink : google::LogSink { severity_char = '?'; break; } + // Set output formatting flags + std::cout << std::setfill('0'); - // 2. Get timestamp components - time_t t = time.time(); - struct tm tm; - localtime_r(&t, &tm); // Thread-safe local time conversion - - // Extract microseconds from the timestamp - int microseconds = static_cast((time.usec() % 1000000)); - - // 3. Get process ID - pid_t pid = getpid(); - - // 格式：级别 + 日期 + 时间 + 进程ID + 文件名:行号] 消息 - std::cout << severity_char << std::setfill('0') << std::setw(4) << tm.tm_year + 1900 // 年份 - << std::setw(2) << tm.tm_mon + 1 // 月份 - << std::setw(2) << tm.tm_mday // 日期 - << " " << std::setw(2) << tm.tm_hour // 小时 - << ":" << std::setw(2) << tm.tm_min // 分钟 - << ":" << std::setw(2) << tm.tm_sec // 秒 - << "." << std::setw(6) << microseconds // 微秒 - << " " << pid // 进程ID - << " " << base_filename // 文件名 - << ":" << line // 行号 - << "] " << std::string(message, message_len) // 日志消息 - << std::endl; + // 1. Log severity (I/W/E/F) + std::cout << severity_char; + + // 2. Date (YYYYMMDD) + // Note: tm_year is years since 1900, tm_mon is 0-based (0-11) + std::cout << std::setw(4) << (time.year() + 1900) << std::setw(2) << std::setfill('0') + << (time.month() + 1) << std::setw(2) << std::setfill('0') << time.day(); + + // 3. Time (HH:MM:SS.ffffff) + std::cout << " " << std::setw(2) << std::setfill('0') << time.hour() << ":" << std::setw(2) + << std::setfill('0') << time.min() << ":" << std::setw(2) << std::setfill('0') + << time.sec() << "." << std::setw(6) << std::setfill('0') << time.usec(); + + // 4. Process ID + std::cout << " " << getpid(); + + // 5. Filename and line number + std::cout << " " << base_filename << ":" << line << "] "; + + // 6. Log message + std::cout.write(message, message_len); + + // Add newline and flush + std::cout << std::endl; } }; From e6eeead716e9f8f792470202e57f50d99428b0c9 Mon Sep 17 00:00:00 2001 From: yiguolei Date: Thu, 31 Jul 2025 10:03:35 +0800 Subject: [PATCH 09/10] f --- be/src/common/logconfig.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/common/logconfig.cpp b/be/src/common/logconfig.cpp index fcec2a0c92f732..05ccc5477d3ad3 100644 --- a/be/src/common/logconfig.cpp +++ b/be/src/common/logconfig.cpp @@ -272,7 +272,7 @@ bool init_glog(const char* basename) { void shutdown_logging() { std::lock_guard logging_lock(logging_mutex); - google::RemoveLogSink(&stdcout_sink); + google::RemoveLogSink(&stdout_log_sink); google::ShutdownGoogleLogging(); } From 0deee9177f6359cb6cfaa7134b695d70c87fba38 Mon Sep 17 00:00:00 2001 From: yiguolei Date: Thu, 31 Jul 2025 12:21:59 +0800 Subject: [PATCH 10/10] f --- bin/start_be.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/start_be.sh b/bin/start_be.sh index 3cd88d11524ed1..f2709cb5a251cd 100755 --- a/bin/start_be.sh +++ b/bin/start_be.sh @@ -107,9 +107,9 @@ log() { cur_date=$(date +"%Y-%m-%d %H:%M:%S,$(date +%3N)") if [[ "${RUN_CONSOLE}" -eq 1 ]]; then echo "StdoutLogger ${cur_date} $1" - else - echo "StdoutLogger ${cur_date} $1" >>"${STDOUT_LOGGER}" fi + # always output start time info into be.out file + echo "StdoutLogger ${cur_date} $1" >>"${STDOUT_LOGGER}" } jdk_version() {