feat(deployment): add datetime output for ob-odc-web.std.log

script/functions.sh

@@ -25,7 +25,7 @@ export ODC_OSS_CONFIG_FILE_NAME=$(echo ~/.odcossutilconfig)
 export ODC_CDN_BASE_URL=${odc_cdn_base_url:-}
 
 function log_info() {
-    echo 1>&2 "$(date +"%Y-%m-%dT%H:%M:%S.%Z") [INFO]" "$*"
+    echo "$(date +"%Y-%m-%dT%H:%M:%S.%Z") [INFO]" "$*"
 }
 
 function log_error() {

script/start-odc.sh

@@ -54,17 +54,25 @@ function usage() {
     echo "- SERVER_PORT: deprecated environment variable, same as ODC_SERVER_PORT, for compatibility"
 }
 
+function log_info() {
+    echo "$(date +"%Y-%m-%dT%H:%M:%S.%Z") [INFO]" "$*"
+}
+
+function log_error() {
+    echo 1>&2 "$(date +"%Y-%m-%dT%H:%M:%S.%Z") [ERROR]" "$*"
+}
+
 function check_env_value_set() {
     local name=$1
     local value=$2
     if [ -z "$value" ]; then
-        echo "FATAL ERROR!, environment variable <${name}> not set, cannot start odc-server"
+        log_error "FATAL ERROR!, environment variable <${name}> not set, cannot start odc-server"
         exit 1
     fi
 }
 
 function init_parameters() {
-    echo "init parameters start"
+    log_info "init parameters start"
 
     # init parameters require environment variable
     check_env_value_set DATABASE_HOST "${DATABASE_HOST}"
@@ -82,31 +90,31 @@ function init_parameters() {
     starter_directory="${ODC_STARTER_DIR:-${install_directory}/starters}"
     obclient_file_path="${OBCLIENT_FILE_PATH:-${install_directory}/obclient/bin/obclient}"
     if [ ! -z "${ODC_HOST}" ]; then
-        echo "ODC_HOST given, will set LOCAL_IP by ODC_HOST, ODC_HOST=${ODC_HOST}"
+        log_info "ODC_HOST given, will set LOCAL_IP by ODC_HOST, ODC_HOST=${ODC_HOST}"
         export LOCAL_IP="${ODC_HOST}"
     else
-        echo "ODC_HOST not given, will set ODC_HOST by ip addr"
-        export LOCAL_IP=`ip addr | grep "eth0" | grep "inet" | awk '{print $2}' | awk -F '/' '{print $1}'`
+        log_info "ODC_HOST not given, will set ODC_HOST by ip addr"
+        export LOCAL_IP=$(ip addr | grep "eth0" | grep "inet" | awk '{print $2}' | awk -F '/' '{print $1}')
     fi
-    export LOCAL_HOSTNAME=`hostname -I | awk -F ' ' '{print $1}'`
+    export LOCAL_HOSTNAME=$(hostname -I | awk -F ' ' '{print $1}')
     export ODC_PROFILE_MODE="${profile}"
-    echo "init parameters done"
+    log_info "init parameters done"
 }
 
 # init jvm args
 function init_jvm_options() {
-    echo "init jvm options start"
+    log_info "init jvm options start"
     heap_options=${ODC_JVM_HEAP_OPTIONS:-${default_heap_options}}
     gc_options=${ODC_JVM_GC_OPTIONS:-${default_gc_options}}
     init_remote_debug_options
     oom_options=${ODC_JVM_OOM_OPTIONS:-${default_oom_options}}
     extra_options="${ODC_JVM_EXTRA_OPTIONS}"
     if [ -z "${SPACEV_JAVA_AGENT}" ]; then
         spacev_java_agent_options=""
-        echo "SPACEV_JAVA_AGENT is not set"
+        log_info "SPACEV_JAVA_AGENT is not set"
     else
         spacev_java_agent_options="${SPACEV_JAVA_AGENT}"
-        echo "SPACEV_JAVA_AGENT is set"
+        log_info "SPACEV_JAVA_AGENT is set"
     fi
     local log_options="-Dlog4j.configurationFile=${app_log_config_file} -Dodc.log.directory=${app_log_directory}"
     local work_dir_options="-Duser.dir=${ODC_WORK_DIR:-${current_work_directory}}"
@@ -121,41 +129,41 @@ function init_jvm_options() {
     local extra_args="${ODC_APP_EXTRA_ARGS}"
     app_args="${listen_port_args} ${obclient_args} ${file_args} ${extra_args}"
 
-    echo "init jvm options done"
+    log_info "init jvm options done"
 }
 
 function init_remote_debug_options() {
     if [ -z "${ODC_REMOTE_DEBUG_PORT}" ]; then
-        echo "ODC_REMOTE_DEBUG_PORT not set, will disable remote debug."
+        log_info "ODC_REMOTE_DEBUG_PORT not set, will disable remote debug."
     else
-        echo "ODC_REMOTE_DEBUG_PORT is set to ${ODC_REMOTE_DEBUG_PORT}, will enable remote debug."
+        log_info "ODC_REMOTE_DEBUG_PORT is set to ${ODC_REMOTE_DEBUG_PORT}, will enable remote debug."
         remote_debug_options="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=${ODC_REMOTE_DEBUG_PORT}"
     fi
 }
 
 function init_java_exec() {
-    echo "init java exec start"
+    log_info "init java exec start"
     try_use_jdk=${ODC_JVM_TRY_USE_JDK:-0}
     java_exec=java
     if [ ! -z "${JAVA_HOME}" ]; then
-        echo "JAVA_HOME detected, will use ${JAVA_HOME}/bin/java instead"
+        log_info "JAVA_HOME detected, will use ${JAVA_HOME}/bin/java instead"
         java_exec="${JAVA_HOME}/bin/java"
     elif [ "1" = "${try_use_jdk}" ]; then
-        echo "ODC_JVM_TRY_USE_JDK detected, try detect jdk home directory..."
+        log_info "ODC_JVM_TRY_USE_JDK detected, try detect jdk home directory..."
         local jdk_home=$(dirname $(dirname $(dirname $(readlink -f $(which java)))))
         local has_jdk=$(if [ -f "${jdk_home}/bin/java" ]; then echo 1; else echo 0; fi)
         if [ "1" = "${has_jdk}" ]; then
             export JAVA_HOME=${jdk_home}
-            echo "jdk_home detected, set as JAVA_HOME, JAVA_HOME=${JAVA_HOME}"
+            log_info "jdk_home detected, set as JAVA_HOME, JAVA_HOME=${JAVA_HOME}"
             java_exec="${JAVA_HOME}/bin/java"
         fi
     fi
     ${java_exec} -version
     if [ $? != 0 ]; then
-        echo "FATAL ERROR! java program <${java_exec}> not found, cannot start odc-server"
+        log_error "FATAL ERROR! java program <${java_exec}> not found, cannot start odc-server"
         exit 1
     fi
-    echo "init java exec done, java_exec=${java_exec}"
+    log_info "init java exec done, java_exec=${java_exec}"
 }
 
 main() {
@@ -169,11 +177,11 @@ main() {
     init_jvm_options
 
     if [ ! -e ${jar_file} ]; then
-        echo "FATAL ERROR!, jar file <${jar_file}> not found, cannot start odc-server"
+        log_error "FATAL ERROR!, jar file <${jar_file}> not found, cannot start odc-server"
         exit 1
     fi
 
-    echo "Starting odc-server..."
+    log_info "Starting odc-server..."
 
     export ODC_DATABASE_HOST=${DATABASE_HOST}
     export ODC_DATABASE_PORT=${DATABASE_PORT}
@@ -184,7 +192,7 @@ main() {
     local cmd="${java_exec} ${remote_debug_options} ${spacev_java_agent_options} ${gc_options} ${heap_options} ${oom_options}
     ${extra_options} ${app_options} -jar
     ${jar_file} ${app_args}"
-    echo "cmd=${cmd}"
+    log_info "cmd=${cmd}"
     eval ${cmd}
     return $?
 }

server/odc-common/src/main/java/com/oceanbase/odc/common/security/SensitiveDataUtils.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2023 OceanBase.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.oceanbase.odc.common.security;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import com.oceanbase.odc.common.util.StringUtils;
+
+public class SensitiveDataUtils {
+
+    private static final Pattern SENSITIVE_PATTERN =
+            Pattern.compile("(secret|key|password|pswd|email|-p)([=|:|\\\"\\s]*)([^&,\\n\\t\\\"]+)",
+                    Pattern.CASE_INSENSITIVE);
+    private static final String MASKED_VALUE = "***";
+
+    public static String mask(String message) {
+        if (message == null || message.isEmpty()) {
+            return message;
+        }
+        try {
+            Matcher matcher = SENSITIVE_PATTERN.matcher(message);
+            if (matcher.find()) {
+                StringBuffer sb = new StringBuffer();
+                do {
+                    matcher.appendReplacement(sb, matcher.group(1) + matcher.group(2) + MASKED_VALUE);
+                } while (matcher.find());
+                matcher.appendTail(sb);
+                return sb.toString();
+            }
+            return message;
+        } catch (Exception ex) {
+            return "MESSAGE_MASK_FAILED, origin message start with " + StringUtils.substring(message, 0, 10);
+        }
+    }
+}

server/odc-common/src/test/java/com/oceanbase/odc/common/security/SensitiveDataUtilsTest.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2023 OceanBase.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.oceanbase.odc.common.security;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class SensitiveDataUtilsTest {
+    @Parameter(0)
+    public String message;
+    @Parameter(1)
+    public String expectedMasked;
+
+    @Parameters(name = "{index}: masked value for {0} expected {1}")
+    public static Collection<Object[]> data() {
+        return Arrays.asList(new Object[][] {
+                {"nomask", "nomask"},
+                {"password=123456", "password=***"},
+                {"-p 123456", "-p ***"},
+                {"-p123456", "-p***"},
+                {"somepassword=123456", "somepassword=***"},
+                {"PASSWORD=123456", "PASSWORD=***"},
+                {"password = 123456", "password = ***"},
+                {"password=123456,secret=654321", "password=***,secret=***"},
+                {"\"password\"=\"123456\"", "\"password\"=\"***\""},
+                {"\"password\" : \"123456\"", "\"password\" : \"***\""},
+                {"hello:{\"password\"=\"123456\",\"secret\"=\"654321\"}",
+                        "hello:{\"password\"=\"***\",\"secret\"=\"***\"}"}
+        });
+    }
+
+    @Test
+    public void mask() {
+        String masked = SensitiveDataUtils.mask(message);
+        Assert.assertEquals(expectedMasked, masked);
+    }
+}

server/odc-server/src/main/java/com/oceanbase/odc/server/OdcServer.java

@@ -36,6 +36,7 @@
 import org.springframework.web.servlet.config.annotation.EnableWebMvc;
 
 import com.oceanbase.odc.common.json.JsonUtils;
+import com.oceanbase.odc.common.security.SensitiveDataUtils;
 import com.oceanbase.odc.common.util.SystemUtils;
 import com.oceanbase.odc.core.authority.interceptor.MethodAuthorizedPostProcessor;
 import com.oceanbase.odc.migrate.AbstractMetaDBMigrate;
@@ -79,9 +80,9 @@ private static void initEnv() {
         log.info("odc server initializing...");
 
         Map<String, String> systemEnv = SystemUtils.getSystemEnv();
-        log.info("systemEnv:\n{}", JsonUtils.prettyToJson(systemEnv));
+        log.info("systemEnv:\n{}", SensitiveDataUtils.mask(JsonUtils.prettyToJson(systemEnv)));
         Properties systemProperties = SystemUtils.getSystemProperties();
-        log.info("systemProperties:\n{}", JsonUtils.prettyToJson(systemProperties));
+        log.info("systemProperties:\n{}", SensitiveDataUtils.mask(JsonUtils.prettyToJson(systemProperties)));
 
         Runtime.getRuntime().addShutdownHook(new Thread(
                 () -> log.info("Oceanbase Developer Center exits, systemInfo={}", SystemUtils.getSystemMemoryInfo())));