From a1563c7d3226dc6fd5fe493cddf0e07c5208794c Mon Sep 17 00:00:00 2001 From: jon Date: Wed, 12 Aug 2020 12:21:18 +0800 Subject: [PATCH] Add setup wizard and optimize quick start --- odps-data-carrier/bin/configure | 209 ++++++++++++++++++ odps-data-carrier/bin/hive_to_mc_quickstart | 91 -------- odps-data-carrier/bin/mma-client | 6 +- odps-data-carrier/bin/mma-server | 7 +- odps-data-carrier/bin/quickstart | 91 ++++++++ odps-data-carrier/build.py | 9 +- ...ve_config.ini => hive_config.ini.template} | 0 odps-data-carrier/mma_client_log4j2.xml | 4 +- odps-data-carrier/mma_server_log4j2.xml | 8 +- ...ps_config.ini => odps_config.ini.template} | 0 ...oss_config.ini => oss_config.ini.template} | 0 .../taskscheduler/MmaClientMain.java | 18 +- .../taskscheduler/MmaConfigUtils.java | 22 +- .../taskscheduler/MmaServerMain.java | 19 +- 14 files changed, 349 insertions(+), 135 deletions(-) create mode 100755 odps-data-carrier/bin/configure delete mode 100644 odps-data-carrier/bin/hive_to_mc_quickstart create mode 100644 odps-data-carrier/bin/quickstart rename odps-data-carrier/{hive_config.ini => hive_config.ini.template} (100%) rename odps-data-carrier/{odps_config.ini => odps_config.ini.template} (100%) rename odps-data-carrier/{oss_config.ini => oss_config.ini.template} (100%) diff --git a/odps-data-carrier/bin/configure b/odps-data-carrier/bin/configure new file mode 100755 index 00000000..d8374e31 --- /dev/null +++ b/odps-data-carrier/bin/configure @@ -0,0 +1,209 @@ +#!/bin/bash + +CUR_DIR=$(pwd) +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +export MMA_HOME=$(dirname ${DIR}) + +# directories +HIVE_CONFIG_PATH="${MMA_HOME}/conf/hive_config.ini" +MC_CONFIG_PATH="${MMA_HOME}/conf/odps_config.ini" + +# constants +# hive metastore uri +HIVE_METASTORE_URIS_PROMPT_ZH="请输入Hive metastore URI(s)" +HIVE_METASTORE_URIS_HELP_ZH="见hive-site.xml中\"hive.metastore.uris\"" +HIVE_METASTORE_URIS_EXAMPLE="thrift://hostname:9083" +# hive jdbc connection url +HIVE_JDBC_CONN_URL_PROMPT_ZH="请输入Hive JDBC连接串" +HIVE_JDBC_CONN_URL_HELP_ZH="即通常通过beeline使用Hive时输入的JDBC连接串, 前缀为jdbc:hive2" +HIVE_JDBC_CONN_URL_EXAMPLE="jdbc:hive2://hostname:10000/default" +# hive jdbc connection user +HIVE_JDBC_CONN_USER_PROMPT_ZH="请输入Hive JDBC连接用户名" +HIVE_JDBC_CONN_USER_HELP_ZH="即通常通过beeline使用Hive时输入的JDBC连接用户名, 默认值为Hive" +HIVE_JDBC_CONN_USER_EXAMPLE="Hive" +# hive jdbc connection password +HIVE_JDBC_CONN_PW_PROMPT_ZH="请输入Hive JDBC连接密码" +HIVE_JDBC_CONN_PW_HELP_ZH="即通常通过beeline使用Hive时输入的JDBC连接密码, 默认值为空" +HIVE_JDBC_CONN_PW_EXAMPLE="" +# mc endpoint +MC_ENDPOINT_PROMPT_ZH="请输入MaxCompute endpoint" +MC_ENDPOINT_HELP_ZH="目标MaxCompute project所在region的endpoint, 详见: https://help.aliyun.com/document_detail/34951.html" +MC_ENDPOINT_EXAMPLE="http://service.cn-hangzhou.maxcompute.aliyun.com/api" +# mc project name +MC_PROJECT_NAME_PROMPT_ZH="请输入MaxCompute project名" +MC_PROJECT_NAME_HELP_ZH="建议配置为目标MaxCompute project, 规避权限问题" +MC_PROJECT_NAME_EXAMPLE="" +# aliyun access id +ALIYUN_ACCESS_ID_PROMPT_ZH="请输入阿里云accesskey id" +ALIYUN_ACCESS_ID_HELP_ZH="详见: https://help.aliyun.com/document_detail/27803.html" +ALIYUN_ACCESS_ID_EXAMPLE="" +# aliyun access key +ALIYUN_ACCESS_KEY_PROMPT_ZH="请输入阿里云accesskey secret" +ALIYUN_ACCESS_KEY_HELP_ZH="详见: https://help.aliyun.com/document_detail/27803.html" +ALIYUN_ACCESS_KEY_EXAMPLE="" + +function PRINTLN_STDERR() +{ + printf "${1}\n" 1>&2 +} + +function PRINT_STDERR_BOLD_BLUE() +{ + printf "\033[1;34m${1}\e[0m" 1 >&2 +} + +function PRINTLN_STDERR_BOLD_BLUE() +{ + printf "\033[1;34m${1}\n\e[0m" 1 >&2 +} + +function PRINTLN_STDERR_BOLD_YELLOW() +{ + printf "\033[1;33m${1}\n\e[0m" 1>&2 +} + +function GET_INPUT() +{ + QUESTION=$1 + HELP=$2 + EXAMPLE=$3 + PRINTLN_STDERR "" + PRINTLN_STDERR_BOLD_BLUE "$QUESTION" + if [[ ${HELP} != "" ]] + then + PRINT_STDERR_BOLD_BLUE "HELP: " + PRINTLN_STDERR "${HELP}" + fi + if [[ ${EXAMPLE} != "" ]] + then + PRINT_STDERR_BOLD_BLUE "EXAMPLE: " + PRINTLN_STDERR "${EXAMPLE}" + fi + read -p ">" INPUT + echo ${INPUT} +} + +function EXIT_ON_FAILURE() { + if [[ "$1" -ne 0 ]] + then + PRINTLN_STDERR_BOLD_RED "执行失败" && exit + else + PRINTLN_STDERR_BOLD_BLUE "执行成功" + fi +} + +function CONFIGURE_HIVE_BASIC() +{ + HIVE_METASTORE_URIS=$(GET_INPUT \ + "$HIVE_METASTORE_URIS_PROMPT_ZH" "$HIVE_METASTORE_URIS_HELP_ZH" "$HIVE_METASTORE_URIS_EXAMPLE") + echo "hms_thrift_addr=${HIVE_METASTORE_URIS}" >> ${HIVE_CONFIG_PATH} + HIVE_JDBC_CONN_URL=$(GET_INPUT \ + "$HIVE_JDBC_CONN_URL_PROMPT_ZH" "$HIVE_JDBC_CONN_URL_HELP_ZH" "$HIVE_JDBC_CONN_URL_EXAMPLE") + echo "jdbc_connection_url=${HIVE_JDBC_CONN_URL}" >> ${HIVE_CONFIG_PATH} + HIVE_JDBC_CONN_USER=$(GET_INPUT \ + "$HIVE_JDBC_CONN_USER_PROMPT_ZH" "$HIVE_JDBC_CONN_USER_HELP_ZH" "$HIVE_JDBC_CONN_USER_EXAMPLE") + echo "user=${HIVE_JDBC_CONN_USER}" >> ${HIVE_CONFIG_PATH} + HIVE_JDBC_CONN_PW=$(GET_INPUT \ + "$HIVE_JDBC_CONN_PW_PROMPT_ZH" "$HIVE_JDBC_CONN_PW_HELP_ZH" "$HIVE_JDBC_CONN_PW_EXAMPLE") + echo "password=${HIVE_JDBC_CONN_PW}" >> ${HIVE_CONFIG_PATH} +} + +function CONFIGURE_HIVE_SECURITY() +{ + # TODO + pass +} + +function PRINT_CONFIGURE_HIVE_UDTF_GUIDANCE() +{ + PRINTLN_STDERR_BOLD_BLUE "上传MaxCompute配置文件至HDFS, 命令为: " + PRINTLN_STDERR "hdfs dfs -put -f ${MMA_HOME}/conf/odps_config.ini hdfs:///tmp/" + PRINTLN_STDERR_BOLD_BLUE "上传Hive UDTF jar包至HDFS, 命令为: " + PRINTLN_STDERR "hdfs dfs -put -f ${MMA_HOME}/lib/data-transfer-hive-udtf-1.0-SNAPSHOT-jar-with-dependencies.jar hdfs:///tmp/" + PRINTLN_STDERR_BOLD_BLUE "创建Hive永久函数, 注意需要通过beeline创建, beeline中命令为: " + PRINTLN_STDERR "CREATE FUNCTION odps_data_dump_multi as 'com.aliyun.odps.datacarrier.transfer.OdpsDataTransferUDTF' USING JAR 'hdfs:///tmp/data-transfer-hive-udtf-1.0-SNAPSHOT-jar-with-dependencies.jar';" +} + +function CONFIGURE_MC() +{ + MC_ENDPOINT=$(GET_INPUT "$MC_ENDPOINT_PROMPT_ZH" "$MC_ENDPOINT_HELP_ZH" "$MC_ENDPOINT_EXAMPLE") + echo "end_point=${MC_ENDPOINT}" >> ${MC_CONFIG_PATH} + MC_PROJECT_NAME=$(GET_INPUT \ + "$MC_PROJECT_NAME_PROMPT_ZH" "$MC_PROJECT_NAME_HELP_ZH" "$MC_PROJECT_NAME_EXAMPLE") + echo "project_name=${MC_PROJECT_NAME}" >> ${MC_CONFIG_PATH} + ALIYUN_ACCESS_ID=$(GET_INPUT \ + "$ALIYUN_ACCESS_ID_PROMPT_ZH" "$ALIYUN_ACCESS_ID_HELP_ZH" "$ALIYUN_ACCESS_ID_EXAMPLE") + echo "access_id=${ALIYUN_ACCESS_ID}" >> ${MC_CONFIG_PATH} + ALIYUN_ACCESS_KEY=$(GET_INPUT \ + "$ALIYUN_ACCESS_KEY_PROMPT_ZH" "$ALIYUN_ACCESS_KEY_HELP_ZH" "$ALIYUN_ACCESS_KEY_EXAMPLE") + echo "access_key=${ALIYUN_ACCESS_KEY}" >> ${MC_CONFIG_PATH} +} + +# Hive basic configurations +PRINTLN_STDERR_BOLD_YELLOW "开始Hive配置" +if [[ -f ${HIVE_CONFIG_PATH} ]] +then + while true; do + DELETE=$(GET_INPUT "Hive配置文件已存在, 是否跳过 (Y/N)" "" "") + case ${DELETE} in + [Yy]) break;; + [Nn]) rm -rf ${HIVE_CONFIG_PATH}; touch ${HIVE_CONFIG_PATH}; CONFIGURE_HIVE_BASIC; break;; + *) PRINTLN_STDERR "请输入'Y'或'N'";; + esac + done +else + touch ${HIVE_CONFIG_PATH} + CONFIGURE_HIVE_BASIC +fi + +# Hive security configurations +while true; do + HAS_SECURITY_CONFIG=$(GET_INPUT "是否有Kerberos配置 (Y/N)" "" "") + case ${HAS_SECURITY_CONFIG} in + [Yy]) CONFIGURE_HIVE_SECURITY; break;; + [Nn]) break;; + *) PRINTLN_STDERR "请输入'Y'或'N'";; + esac +done + +# MaxCompute configurations +PRINTLN_STDERR "" +PRINTLN_STDERR_BOLD_YELLOW "开始MaxCompute配置" +if [[ -f ${MC_CONFIG_PATH} ]] +then + while true; do + DELETE=$(GET_INPUT "MaxCompute配置文件已存在, 是否跳过 (Y/N)" "" "") + case ${DELETE} in + [Yy]) break;; + [Nn]) rm -rf ${MC_CONFIG_PATH}; touch ${MC_CONFIG_PATH}; CONFIGURE_MC; break;; + *) PRINTLN_STDERR "请输入'Y'或'N'";; + esac + done +else + touch ${MC_CONFIG_PATH} + CONFIGURE_MC +fi + +PRINTLN_STDERR "" +PRINTLN_STDERR_BOLD_YELLOW "正在生成MMA配置" +cd ${MMA_HOME}/conf/ +sh ${MMA_HOME}/bin/generate-config -s \ + --hive_config ${MMA_HOME}/conf/hive_config.ini \ + --odps_config ${MMA_HOME}/conf/odps_config.ini > /dev/null 2>&1 +EXIT_ON_FAILURE $? +cp ${MMA_HOME}/conf/mma_server_config.json ${MMA_HOME}/conf/mma_client_config.json +PRINTLN_STDERR_BOLD_YELLOW "MMA server默认配置路径: ${MMA_HOME}/conf/mma_server_config.json" +PRINTLN_STDERR_BOLD_YELLOW "MMA client默认配置路径: ${MMA_HOME}/conf/mma_client_config.json" + +PRINTLN_STDERR "" +PRINTLN_STDERR_BOLD_YELLOW "创建数据传输所需Hive UDTF, 以下命令需要您手动执行" +PRINT_CONFIGURE_HIVE_UDTF_GUIDANCE + +while true; do + FINISHED=$(GET_INPUT "请确认是否已创建Hive UDTF (Y/N)" "" "") + case ${FINISHED} in + [Yy]) PRINTLN_STDERR ""; PRINTLN_STDERR_BOLD_YELLOW "配置完成"; break;; + [Nn]) ;; + *) PRINTLN_STDERR "请输入'Y'或'N'";; + esac +done diff --git a/odps-data-carrier/bin/hive_to_mc_quickstart b/odps-data-carrier/bin/hive_to_mc_quickstart deleted file mode 100644 index 8181c081..00000000 --- a/odps-data-carrier/bin/hive_to_mc_quickstart +++ /dev/null @@ -1,91 +0,0 @@ -#!/bin/bash - -function exit_on_failure() { - if [ "$1" -ne 0 ] - then - echo "执行失败" && exit - else - echo "执行成功" - fi -} - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -MMA_HOME="$DIR/../" - -if [[ $# -lt 4 ]]; then - echo "Usage: sh hive_to_mc_quickstart SOURCE_DB SOURCE_TBL DEST_PROJECT DEST_TBL" - exit -1 -fi - -SOURCE_DB=$1 -SOURCT_TBL=$2 -DEST_DB=$3 -DEST_TBL=$4 - -echo "运行 quick start 前, 请确认您已经完成以下步骤" -echo " 1. 配置 conf 目录下的 odps_odps_config.ini 与 hive_config.ini" -echo " 2. 上传 conf 目录下的 odps_config.ini 至 hdfs:///tmp/odps_config.ini" -echo " 3. 创建永久函数 odps_data_dump_multi" -echo "quick start 将于 5 秒后开始运行, 输入 Ctrl + C 取消执行" - -sleep 5 - -echo "进入目录 ${MMA_HOME}" -cd "${MMA_HOME}" || echo "执行失败" || exit - -echo "" -echo "生成 MMA server 配置文件, 执行命令: " -echo " sh bin/generate-config --to_server_config \\" -echo " --hive_config conf/hive_config.ini \\" -echo " --odps_config conf/odps_config.ini > /dev/null 2>&1" -sh bin/generate-config --to_server_config \ ---hive_config conf/hive_config.ini \ ---odps_config conf/odps_config.ini > /dev/null 2>&1 -exit_on_failure $? - -echo "" -echo "生成 MMA client 配置文件, 执行命令: " -echo " sh bin/generate-config --to_client_config \\" -echo " --hive_config conf/hive_config.ini > /dev/null 2>&1" -sh bin/generate-config --to_client_config \ ---hive_config conf/hive_config.ini > /dev/null 2>&1 -exit_on_failure $? - -echo "" -echo "生成 MMA 迁移任务描述文件, 执行命令: " -echo " echo $SOURCE_DB.$SOURCT_TBL:$DEST_DB.$DEST_TBL > table_mapping.txt" -echo "$SOURCE_DB.$SOURCT_TBL:$DEST_DB.$DEST_TBL" > conf/table_mapping.txt -echo " sh bin/generate-config --to_migration_config \\" -echo " --table_mapping conf/table_mapping.txt > /dev/null 2>&1" -sh bin/generate-config --to_migration_config \ ---table_mapping conf/table_mapping.txt > /dev/null 2>&1 -exit_on_failure $? - -echo "" -echo "启动 MMA server, 执行命令: " -echo " sh odps-data-carrier/bin/mma-server \ " -echo " --config configs/mma_server_config.json > /dev/null 2>&1 &" -sh bin/mma-server --config mma_server_config.json /dev/null 2>&1 & -PID=$(jps | grep MmaServerMain | awk '{print $1}') -echo "MMA server pid: ${PID}" - -echo "" -echo "提交迁移任务, 执行命令: " -echo " sh bin/mma-client \\" -echo " --config mma_client_config.json --start mma_migration_config.json" -sh bin/mma-client \ ---config mma_server_config.json --start mma_migration_config.json -exit_on_failure $? - -echo "" -echo "等待任务结束, 执行命令: " -echo " sh bin/mma-client \\" -echo " --config mma_server_config.json --wait $SOURCE_DB.$SOURCT_TBL" -sh bin/mma-client --config mma_server_config.json --wait "$SOURCE_DB.$SOURCT_TBL" -exit_on_failure $? - -echo "" -echo "迁移完成, 结束 MMA server, 执行命令: " -echo " kill ${PID}" -kill "${PID}" -exit_on_failure $? diff --git a/odps-data-carrier/bin/mma-client b/odps-data-carrier/bin/mma-client index d62a6ad6..044583ca 100755 --- a/odps-data-carrier/bin/mma-client +++ b/odps-data-carrier/bin/mma-client @@ -1,5 +1,7 @@ #!/bin/bash DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -export MMA_HOME="$DIR/../" +export MMA_HOME=$(dirname ${DIR}) -java -Dlog4j.configurationFile="$MMA_HOME/res/mma_client_log4j2.xml" -cp "$MMA_HOME/lib/*" com.aliyun.odps.datacarrier.taskscheduler.MmaClientMain "$@" +java -Dlog4j.configurationFile="$MMA_HOME/res/mma_client_log4j2.xml" \ + -cp "$MMA_HOME/lib/*" \ + com.aliyun.odps.datacarrier.taskscheduler.MmaClientMain "$@" diff --git a/odps-data-carrier/bin/mma-server b/odps-data-carrier/bin/mma-server index fe896206..6aab9467 100755 --- a/odps-data-carrier/bin/mma-server +++ b/odps-data-carrier/bin/mma-server @@ -1,6 +1,7 @@ #!/bin/bash DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -export MMA_HOME="$DIR/../" +export MMA_HOME=$(dirname ${DIR}) -# TODO: ensure singleton -java -Dlog4j.configurationFile="$MMA_HOME/res/mma_server_log4j2.xml" -cp "$MMA_HOME/lib/*" com.aliyun.odps.datacarrier.taskscheduler.MmaServerMain "$@" +java -Dlog4j.configurationFile="$MMA_HOME/res/mma_server_log4j2.xml" \ + -cp "$MMA_HOME/lib/*" \ + com.aliyun.odps.datacarrier.taskscheduler.MmaServerMain "$@" diff --git a/odps-data-carrier/bin/quickstart b/odps-data-carrier/bin/quickstart new file mode 100644 index 00000000..e3ea1670 --- /dev/null +++ b/odps-data-carrier/bin/quickstart @@ -0,0 +1,91 @@ +#!/bin/bash + +set -e +CUR_DIR=$(pwd) +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +export MMA_HOME=$(dirname ${DIR}) + +function PRINTLN_STDERR() +{ + printf "${1}\n" 1>&2 +} + +function PRINT_STDERR_BOLD_BLUE() +{ + printf "\033[1;34m${1}\e[0m" 1 >&2 +} + +function PRINTLN_STDERR_BOLD_BLUE() +{ + printf "\033[1;34m${1}\n\e[0m" 1 >&2 +} + +function PRINTLN_STDERR_BOLD_YELLOW() +{ + printf "\033[1;33m${1}\n\e[0m" 1>&2 +} + +function PRINTLN_STDERR_BOLD_RED() +{ + printf "\033[1;31m${1}\n\e[0m" 1>&2 +} + +function EXIT_ON_FAILURE() { + if [[ "$1" -ne 0 ]] + then + PRINTLN_STDERR_BOLD_RED "执行失败" && exit + else + PRINTLN_STDERR_BOLD_BLUE "执行成功" + fi +} + +if [[ $# -lt 4 ]]; then + PRINTLN_STDERR_BOLD_YELLOW "Usage: sh quickstart SOURCE_DB SOURCE_TBL DEST_PROJECT DEST_TBL" + exit -1 +fi + +SOURCE_DB=$1 +SOURCE_TBL=$2 +DEST_DB=$3 +DEST_TBL=$4 + +PRINTLN_STDERR_BOLD_YELLOW "运行 quick start 前, 请确认您已经完成以下步骤" +PRINTLN_STDERR_BOLD_YELLOW " 1. 执行configure, 按照引导生成MMA所需配置文件, 并创建Hive永久函数" +PRINTLN_STDERR_BOLD_YELLOW "quick start 将于 5 秒后开始运行, 输入 Ctrl + C 取消执行" + +sleep 5 + +PRINTLN_STDERR_BOLD_YELLOW "" +PRINTLN_STDERR_BOLD_YELLOW "生成 MMA 迁移任务描述文件, 执行命令: " +PRINTLN_STDERR_BOLD_YELLOW " echo ${SOURCE_DB}.${SOURCE_TBL}:${DEST_DB}.${DEST_TBL} > table_mapping.txt" +echo ${SOURCE_DB}.${SOURCE_TBL}:${DEST_DB}.${DEST_TBL} > table_mapping.txt +PRINTLN_STDERR_BOLD_YELLOW " sh ${MMA_HOME}/bin/generate-config --to_migration_config \\" +PRINTLN_STDERR_BOLD_YELLOW " --table_mapping table_mapping.txt > /dev/null 2>&1" +sh ${MMA_HOME}/bin/generate-config --to_migration_config \ +--table_mapping table_mapping.txt > /dev/null 2>&1 +EXIT_ON_FAILURE $? + +PRINTLN_STDERR_BOLD_YELLOW "" +PRINTLN_STDERR_BOLD_YELLOW "启动 MMA server, 执行命令: " +PRINTLN_STDERR_BOLD_YELLOW " sh ${MMA_HOME}/bin/mma-server > /dev/null 2>&1 &" +sh ${MMA_HOME}/bin/mma-server > /dev/null 2>&1 & +PID=$(jps | grep MmaServerMain | awk '{print $1}') +PRINTLN_STDERR_BOLD_YELLOW "MMA server pid: ${PID}" + +PRINTLN_STDERR_BOLD_YELLOW "" +PRINTLN_STDERR_BOLD_YELLOW "提交迁移任务, 执行命令: " +PRINTLN_STDERR_BOLD_YELLOW " sh ${MMA_HOME}/bin/mma-client --start mma_migration_config.json" +sh ${MMA_HOME}/bin/mma-client --start mma_migration_config.json +EXIT_ON_FAILURE $? + +PRINTLN_STDERR_BOLD_YELLOW "" +PRINTLN_STDERR_BOLD_YELLOW "等待任务结束, 执行命令: " +PRINTLN_STDERR_BOLD_YELLOW " sh ${MMA_HOME}/bin/mma-client --wait $SOURCE_DB.$SOURCE_TBL" +sh ${MMA_HOME}/bin/mma-client --wait "$SOURCE_DB.$SOURCE_TBL" +EXIT_ON_FAILURE $? + +PRINTLN_STDERR_BOLD_YELLOW "" +PRINTLN_STDERR_BOLD_YELLOW "迁移完成, 结束 MMA server, 执行命令: " +PRINTLN_STDERR_BOLD_YELLOW " kill ${PID}" +kill "${PID}" +EXIT_ON_FAILURE $? \ No newline at end of file diff --git a/odps-data-carrier/build.py b/odps-data-carrier/build.py index 6d7478a4..28909ef1 100644 --- a/odps-data-carrier/build.py +++ b/odps-data-carrier/build.py @@ -85,7 +85,8 @@ def execute(cmd: str, verbose=False) -> int: shutil.copyfile("bin/mma-server", "odps-data-carrier/bin/mma-server") shutil.copyfile("bin/mma-client", "odps-data-carrier/bin/mma-client") shutil.copyfile("bin/generate-config", "odps-data-carrier/bin/generate-config") - shutil.copyfile("bin/hive_to_mc_quickstart", "odps-data-carrier/bin/hive_to_mc_quickstart") + shutil.copyfile("bin/configure", "odps-data-carrier/bin/configure") + shutil.copyfile("bin/quickstart", "odps-data-carrier/bin/quickstart") # lib task_scheduler_jar_name = "task-scheduler-1.0-SNAPSHOT.jar" @@ -95,9 +96,9 @@ def execute(cmd: str, verbose=False) -> int: "odps-data-carrier/lib/" + udtf_jar_name) # conf - shutil.copyfile("odps_config.ini", "odps-data-carrier/conf/odps_config.ini") - shutil.copyfile("oss_config.ini", "odps-data-carrier/conf/oss_config.ini") - shutil.copyfile("hive_config.ini", "odps-data-carrier/conf/hive_config.ini") + shutil.copyfile("odps_config.ini.template", "odps-data-carrier/conf/odps_config.ini.template") + shutil.copyfile("oss_config.ini.template", "odps-data-carrier/conf/oss_config.ini.template") + shutil.copyfile("hive_config.ini.template", "odps-data-carrier/conf/hive_config.ini.template") shutil.copyfile("table_mapping.txt", "odps-data-carrier/conf/table_mapping.txt") # res diff --git a/odps-data-carrier/hive_config.ini b/odps-data-carrier/hive_config.ini.template similarity index 100% rename from odps-data-carrier/hive_config.ini rename to odps-data-carrier/hive_config.ini.template diff --git a/odps-data-carrier/mma_client_log4j2.xml b/odps-data-carrier/mma_client_log4j2.xml index aa257ee9..5d9133c9 100644 --- a/odps-data-carrier/mma_client_log4j2.xml +++ b/odps-data-carrier/mma_client_log4j2.xml @@ -1,8 +1,8 @@ - + %d{DEFAULT} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n diff --git a/odps-data-carrier/mma_server_log4j2.xml b/odps-data-carrier/mma_server_log4j2.xml index 80ef7904..9240d6fd 100644 --- a/odps-data-carrier/mma_server_log4j2.xml +++ b/odps-data-carrier/mma_server_log4j2.xml @@ -1,8 +1,8 @@ - + %d{DEFAULT} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n @@ -16,8 +16,8 @@ - + %m%n diff --git a/odps-data-carrier/odps_config.ini b/odps-data-carrier/odps_config.ini.template similarity index 100% rename from odps-data-carrier/odps_config.ini rename to odps-data-carrier/odps_config.ini.template diff --git a/odps-data-carrier/oss_config.ini b/odps-data-carrier/oss_config.ini.template similarity index 100% rename from odps-data-carrier/oss_config.ini rename to odps-data-carrier/oss_config.ini.template diff --git a/odps-data-carrier/task-scheduler/src/main/java/com/aliyun/odps/datacarrier/taskscheduler/MmaClientMain.java b/odps-data-carrier/task-scheduler/src/main/java/com/aliyun/odps/datacarrier/taskscheduler/MmaClientMain.java index 95b4cc85..ea60e828 100644 --- a/odps-data-carrier/task-scheduler/src/main/java/com/aliyun/odps/datacarrier/taskscheduler/MmaClientMain.java +++ b/odps-data-carrier/task-scheduler/src/main/java/com/aliyun/odps/datacarrier/taskscheduler/MmaClientMain.java @@ -230,7 +230,6 @@ private static int remove(MmaClient client, String jobName) { */ private static int list(MmaClient client, String statusStr) { MmaMetaManager.MigrationStatus status; - // TODO: list all should output migration job along with its status if ("ALL".equalsIgnoreCase(statusStr)) { status = null; } else { @@ -276,15 +275,18 @@ private static int list(MmaClient client, String statusStr) { public static void main(String[] args) throws ParseException, IOException, MetaException { BasicConfigurator.configure(); - /* - Required options - */ + + String mmaHome = System.getenv("MMA_HOME"); + if (mmaHome == null) { + throw new IllegalStateException("Environment variable 'MMA_HOME' not set"); + } + Option configOption = Option .builder(CONFIG_OPT) .longOpt(CONFIG_OPT) .argName(CONFIG_OPT) .hasArg() - .desc("MMA client configuration, required") + .desc("MMA client configuration") .build(); /* Sub commands, mutually exclusive @@ -351,11 +353,13 @@ public static void main(String[] args) throws ParseException, IOException, MetaE System.exit(help(options)); } + Path mmaClientConfigFilePath; if (!cmd.hasOption(CONFIG_OPT)) { - throw new IllegalArgumentException("Required argument 'config'"); + mmaClientConfigFilePath = Paths.get(mmaHome, "conf", "mma_client_config.json"); + } else { + mmaClientConfigFilePath = Paths.get(cmd.getOptionValue(CONFIG_OPT)); } - Path mmaClientConfigFilePath = Paths.get(cmd.getOptionValue(CONFIG_OPT)); MmaServerConfig.init(mmaClientConfigFilePath); // Check if more than one sub command is given diff --git a/odps-data-carrier/task-scheduler/src/main/java/com/aliyun/odps/datacarrier/taskscheduler/MmaConfigUtils.java b/odps-data-carrier/task-scheduler/src/main/java/com/aliyun/odps/datacarrier/taskscheduler/MmaConfigUtils.java index c974a444..462c12a3 100644 --- a/odps-data-carrier/task-scheduler/src/main/java/com/aliyun/odps/datacarrier/taskscheduler/MmaConfigUtils.java +++ b/odps-data-carrier/task-scheduler/src/main/java/com/aliyun/odps/datacarrier/taskscheduler/MmaConfigUtils.java @@ -50,6 +50,8 @@ import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; +import org.apache.commons.configuration.BaseConfiguration; +import org.apache.log4j.BasicConfigurator; import org.apache.tools.ant.filters.StringInputStream; import com.aliyun.odps.datacarrier.taskscheduler.MmaConfig.AdditionalTableConfig; @@ -270,11 +272,13 @@ public static void generateSampleConfigs() throws IOException { DEFAULT_ADDITIONAL_TABLE_CONFIG); DirUtils.writeFile(Paths.get("mma_server_config.json"), mmaServerConfig.toJson()); -// DirUtils.writeFile(Paths.get("mma_client_config.json"), mmaClientConfig.toJson()); + DirUtils.writeFile(Paths.get("mma_client_config.json"), mmaServerConfig.toJson()); DirUtils.writeFile(Paths.get("mma_migration_config.json"), mmaMigrationConfig.toJson()); } public static void main(String[] args) throws Exception { + BasicConfigurator.configure(); + Option helpOption = Option .builder("h") .longOpt(HELP) @@ -289,13 +293,6 @@ public static void main(String[] args) throws Exception { .desc("generate sample server, client and migration configs") .build(); - Option clientOption = Option - .builder("c") - .longOpt("to_client_config") - .hasArg(false) - .desc("convert hive_config.ini specified by '--hive_config' to mma client config") - .build(); - Option serverOption = Option .builder("s") .longOpt("to_server_config") @@ -346,7 +343,6 @@ public static void main(String[] args) throws Exception { Options options = new Options() .addOption(helpOption) .addOption(sampleOption) - .addOption(clientOption) .addOption(serverOption) .addOption(migrationOption) .addOption(hiveConfigOption) @@ -372,14 +368,6 @@ public static void main(String[] args) throws Exception { prefix = cmd.getOptionValue("prefix").trim(); } - if (cmd.hasOption("to_client_config")) { - if (!cmd.hasOption("hive_config")) { - throw new IllegalArgumentException("Requires '--hive_config'"); - } - -// generateMmaClientConfig(Paths.get(cmd.getOptionValue("hive_config")), prefix); - } - if (cmd.hasOption("to_server_config")) { if (!cmd.hasOption("hive_config") || !cmd.hasOption("odps_config")) { throw new IllegalArgumentException("Requires '--hive_config' and '--odps_config'"); diff --git a/odps-data-carrier/task-scheduler/src/main/java/com/aliyun/odps/datacarrier/taskscheduler/MmaServerMain.java b/odps-data-carrier/task-scheduler/src/main/java/com/aliyun/odps/datacarrier/taskscheduler/MmaServerMain.java index 70dcd0d5..ef976453 100644 --- a/odps-data-carrier/task-scheduler/src/main/java/com/aliyun/odps/datacarrier/taskscheduler/MmaServerMain.java +++ b/odps-data-carrier/task-scheduler/src/main/java/com/aliyun/odps/datacarrier/taskscheduler/MmaServerMain.java @@ -43,15 +43,18 @@ private static int help(Options options) { public static void main(String[] args) throws ParseException, IOException, MetaException, MmaException { BasicConfigurator.configure(); - /* - Required options - */ + + String mmaHome = System.getenv("MMA_HOME"); + if (mmaHome == null) { + throw new IllegalStateException("Environment variable 'MMA_HOME' not set"); + } + Option configOption = Option .builder(CONFIG_OPT) .longOpt(CONFIG_OPT) .argName(CONFIG_OPT) .hasArg() - .desc("MMA client configuration, required") + .desc("MMA server configuration path") .build(); /* @@ -76,8 +79,14 @@ public static void main(String[] args) System.exit(help(options)); } + Path mmaServerConfigPath; + if (!cmd.hasOption(CONFIG_OPT)) { + mmaServerConfigPath = Paths.get(mmaHome, "conf", "mma_server_config.json"); + } else { + mmaServerConfigPath = Paths.get(cmd.getOptionValue("config")); + } + // Setup MmaServerConfig singleton - Path mmaServerConfigPath = Paths.get(cmd.getOptionValue("config")); MmaServerConfig.init(mmaServerConfigPath); // Setup MmaEventManager singleton