Skip to content

Commit

Permalink
feat(deployment): add datetime output for ob-odc-web.std.log (#1420)
Browse files Browse the repository at this point in the history
* feat(deployment): add datetime output for ob-odc-web.std.log

* refine startup log output

* refine startup log output
  • Loading branch information
yizhouxw committed Jan 19, 2024
1 parent 3965233 commit 0c29f2c
Show file tree
Hide file tree
Showing 5 changed files with 141 additions and 25 deletions.
2 changes: 1 addition & 1 deletion script/functions.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ export ODC_OSS_CONFIG_FILE_NAME=$(echo ~/.odcossutilconfig)
export ODC_CDN_BASE_URL=${odc_cdn_base_url:-}

function log_info() {
echo 1>&2 "$(date +"%Y-%m-%dT%H:%M:%S.%Z") [INFO]" "$*"
echo "$(date +"%Y-%m-%dT%H:%M:%S.%Z") [INFO]" "$*"
}

function log_error() {
Expand Down
52 changes: 30 additions & 22 deletions script/start-odc.sh
Original file line number Diff line number Diff line change
Expand Up @@ -54,17 +54,25 @@ function usage() {
echo "- SERVER_PORT: deprecated environment variable, same as ODC_SERVER_PORT, for compatibility"
}

function log_info() {
echo "$(date +"%Y-%m-%dT%H:%M:%S.%Z") [INFO]" "$*"
}

function log_error() {
echo 1>&2 "$(date +"%Y-%m-%dT%H:%M:%S.%Z") [ERROR]" "$*"
}

function check_env_value_set() {
local name=$1
local value=$2
if [ -z "$value" ]; then
echo "FATAL ERROR!, environment variable <${name}> not set, cannot start odc-server"
log_error "FATAL ERROR!, environment variable <${name}> not set, cannot start odc-server"
exit 1
fi
}

function init_parameters() {
echo "init parameters start"
log_info "init parameters start"

# init parameters require environment variable
check_env_value_set DATABASE_HOST "${DATABASE_HOST}"
Expand All @@ -82,31 +90,31 @@ function init_parameters() {
starter_directory="${ODC_STARTER_DIR:-${install_directory}/starters}"
obclient_file_path="${OBCLIENT_FILE_PATH:-${install_directory}/obclient/bin/obclient}"
if [ ! -z "${ODC_HOST}" ]; then
echo "ODC_HOST given, will set LOCAL_IP by ODC_HOST, ODC_HOST=${ODC_HOST}"
log_info "ODC_HOST given, will set LOCAL_IP by ODC_HOST, ODC_HOST=${ODC_HOST}"
export LOCAL_IP="${ODC_HOST}"
else
echo "ODC_HOST not given, will set ODC_HOST by ip addr"
export LOCAL_IP=`ip addr | grep "eth0" | grep "inet" | awk '{print $2}' | awk -F '/' '{print $1}'`
log_info "ODC_HOST not given, will set ODC_HOST by ip addr"
export LOCAL_IP=$(ip addr | grep "eth0" | grep "inet" | awk '{print $2}' | awk -F '/' '{print $1}')
fi
export LOCAL_HOSTNAME=`hostname -I | awk -F ' ' '{print $1}'`
export LOCAL_HOSTNAME=$(hostname -I | awk -F ' ' '{print $1}')
export ODC_PROFILE_MODE="${profile}"
echo "init parameters done"
log_info "init parameters done"
}

# init jvm args
function init_jvm_options() {
echo "init jvm options start"
log_info "init jvm options start"
heap_options=${ODC_JVM_HEAP_OPTIONS:-${default_heap_options}}
gc_options=${ODC_JVM_GC_OPTIONS:-${default_gc_options}}
init_remote_debug_options
oom_options=${ODC_JVM_OOM_OPTIONS:-${default_oom_options}}
extra_options="${ODC_JVM_EXTRA_OPTIONS}"
if [ -z "${SPACEV_JAVA_AGENT}" ]; then
spacev_java_agent_options=""
echo "SPACEV_JAVA_AGENT is not set"
log_info "SPACEV_JAVA_AGENT is not set"
else
spacev_java_agent_options="${SPACEV_JAVA_AGENT}"
echo "SPACEV_JAVA_AGENT is set"
log_info "SPACEV_JAVA_AGENT is set"
fi
local log_options="-Dlog4j.configurationFile=${app_log_config_file} -Dodc.log.directory=${app_log_directory}"
local work_dir_options="-Duser.dir=${ODC_WORK_DIR:-${current_work_directory}}"
Expand All @@ -121,41 +129,41 @@ function init_jvm_options() {
local extra_args="${ODC_APP_EXTRA_ARGS}"
app_args="${listen_port_args} ${obclient_args} ${file_args} ${extra_args}"

echo "init jvm options done"
log_info "init jvm options done"
}

function init_remote_debug_options() {
if [ -z "${ODC_REMOTE_DEBUG_PORT}" ]; then
echo "ODC_REMOTE_DEBUG_PORT not set, will disable remote debug."
log_info "ODC_REMOTE_DEBUG_PORT not set, will disable remote debug."
else
echo "ODC_REMOTE_DEBUG_PORT is set to ${ODC_REMOTE_DEBUG_PORT}, will enable remote debug."
log_info "ODC_REMOTE_DEBUG_PORT is set to ${ODC_REMOTE_DEBUG_PORT}, will enable remote debug."
remote_debug_options="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=${ODC_REMOTE_DEBUG_PORT}"
fi
}

function init_java_exec() {
echo "init java exec start"
log_info "init java exec start"
try_use_jdk=${ODC_JVM_TRY_USE_JDK:-0}
java_exec=java
if [ ! -z "${JAVA_HOME}" ]; then
echo "JAVA_HOME detected, will use ${JAVA_HOME}/bin/java instead"
log_info "JAVA_HOME detected, will use ${JAVA_HOME}/bin/java instead"
java_exec="${JAVA_HOME}/bin/java"
elif [ "1" = "${try_use_jdk}" ]; then
echo "ODC_JVM_TRY_USE_JDK detected, try detect jdk home directory..."
log_info "ODC_JVM_TRY_USE_JDK detected, try detect jdk home directory..."
local jdk_home=$(dirname $(dirname $(dirname $(readlink -f $(which java)))))
local has_jdk=$(if [ -f "${jdk_home}/bin/java" ]; then echo 1; else echo 0; fi)
if [ "1" = "${has_jdk}" ]; then
export JAVA_HOME=${jdk_home}
echo "jdk_home detected, set as JAVA_HOME, JAVA_HOME=${JAVA_HOME}"
log_info "jdk_home detected, set as JAVA_HOME, JAVA_HOME=${JAVA_HOME}"
java_exec="${JAVA_HOME}/bin/java"
fi
fi
${java_exec} -version
if [ $? != 0 ]; then
echo "FATAL ERROR! java program <${java_exec}> not found, cannot start odc-server"
log_error "FATAL ERROR! java program <${java_exec}> not found, cannot start odc-server"
exit 1
fi
echo "init java exec done, java_exec=${java_exec}"
log_info "init java exec done, java_exec=${java_exec}"
}

main() {
Expand All @@ -169,11 +177,11 @@ main() {
init_jvm_options

if [ ! -e ${jar_file} ]; then
echo "FATAL ERROR!, jar file <${jar_file}> not found, cannot start odc-server"
log_error "FATAL ERROR!, jar file <${jar_file}> not found, cannot start odc-server"
exit 1
fi

echo "Starting odc-server..."
log_info "Starting odc-server..."

export ODC_DATABASE_HOST=${DATABASE_HOST}
export ODC_DATABASE_PORT=${DATABASE_PORT}
Expand All @@ -184,7 +192,7 @@ main() {
local cmd="${java_exec} ${remote_debug_options} ${spacev_java_agent_options} ${gc_options} ${heap_options} ${oom_options}
${extra_options} ${app_options} -jar
${jar_file} ${app_args}"
echo "cmd=${cmd}"
log_info "cmd=${cmd}"
eval ${cmd}
return $?
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
/*
* Copyright (c) 2023 OceanBase.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.oceanbase.odc.common.security;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

import com.oceanbase.odc.common.util.StringUtils;

public class SensitiveDataUtils {

private static final Pattern SENSITIVE_PATTERN =
Pattern.compile("(secret|key|password|pswd|email|-p)([=|:|\\\"\\s]*)([^&,\\n\\t\\\"]+)",
Pattern.CASE_INSENSITIVE);
private static final String MASKED_VALUE = "***";

public static String mask(String message) {
if (message == null || message.isEmpty()) {
return message;
}
try {
Matcher matcher = SENSITIVE_PATTERN.matcher(message);
if (matcher.find()) {
StringBuffer sb = new StringBuffer();
do {
matcher.appendReplacement(sb, matcher.group(1) + matcher.group(2) + MASKED_VALUE);
} while (matcher.find());
matcher.appendTail(sb);
return sb.toString();
}
return message;
} catch (Exception ex) {
return "MESSAGE_MASK_FAILED, origin message start with " + StringUtils.substring(message, 0, 10);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
/*
* Copyright (c) 2023 OceanBase.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.oceanbase.odc.common.security;

import java.util.Arrays;
import java.util.Collection;

import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameter;
import org.junit.runners.Parameterized.Parameters;

@RunWith(Parameterized.class)
public class SensitiveDataUtilsTest {
@Parameter(0)
public String message;
@Parameter(1)
public String expectedMasked;

@Parameters(name = "{index}: masked value for {0} expected {1}")
public static Collection<Object[]> data() {
return Arrays.asList(new Object[][] {
{"nomask", "nomask"},
{"password=123456", "password=***"},
{"-p 123456", "-p ***"},
{"-p123456", "-p***"},
{"somepassword=123456", "somepassword=***"},
{"PASSWORD=123456", "PASSWORD=***"},
{"password = 123456", "password = ***"},
{"password=123456,secret=654321", "password=***,secret=***"},
{"\"password\"=\"123456\"", "\"password\"=\"***\""},
{"\"password\" : \"123456\"", "\"password\" : \"***\""},
{"hello:{\"password\"=\"123456\",\"secret\"=\"654321\"}",
"hello:{\"password\"=\"***\",\"secret\"=\"***\"}"}
});
}

@Test
public void mask() {
String masked = SensitiveDataUtils.mask(message);
Assert.assertEquals(expectedMasked, masked);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import org.springframework.web.servlet.config.annotation.EnableWebMvc;

import com.oceanbase.odc.common.json.JsonUtils;
import com.oceanbase.odc.common.security.SensitiveDataUtils;
import com.oceanbase.odc.common.util.SystemUtils;
import com.oceanbase.odc.core.authority.interceptor.MethodAuthorizedPostProcessor;
import com.oceanbase.odc.migrate.AbstractMetaDBMigrate;
Expand Down Expand Up @@ -79,9 +80,9 @@ private static void initEnv() {
log.info("odc server initializing...");

Map<String, String> systemEnv = SystemUtils.getSystemEnv();
log.info("systemEnv:\n{}", JsonUtils.prettyToJson(systemEnv));
log.info("systemEnv:\n{}", SensitiveDataUtils.mask(JsonUtils.prettyToJson(systemEnv)));
Properties systemProperties = SystemUtils.getSystemProperties();
log.info("systemProperties:\n{}", JsonUtils.prettyToJson(systemProperties));
log.info("systemProperties:\n{}", SensitiveDataUtils.mask(JsonUtils.prettyToJson(systemProperties)));

Runtime.getRuntime().addShutdownHook(new Thread(
() -> log.info("Oceanbase Developer Center exits, systemInfo={}", SystemUtils.getSystemMemoryInfo())));
Expand Down

0 comments on commit 0c29f2c

Please sign in to comment.