diff --git a/docs/admin_guide/install.md b/docs/admin_guide/install.md index eee99e2d7fe4db..a54afc256b677a 100644 --- a/docs/admin_guide/install.md +++ b/docs/admin_guide/install.md @@ -104,7 +104,7 @@ Palo 主要包括 Frontend(FE)和 Backend(BE)两个进程。其中 FE ### 3.3 (可选)FS_Broker 部署 -broker 以插件的形式,独立于 Palo 部署。如果需要从第三方存储系统导入数据,需要部署相应的 broker,默认提供了读取HDFS 和百度云 BOS 的 fs_broker。fs_broker 是无状态的,建议每一个 FE 和 BE 节点都部署一个 broker。 +broker 以插件的形式,独立于 Palo 部署。如果需要从第三方存储系统导入数据,需要部署相应的 broker,默认提供了读取 HDFS 和百度云 BOS 的 fs_broker。fs_broker 是无状态的,建议每一个 FE 和 BE 节点都部署一个 broker。 * 拷贝源码 fs_broker 的 output 目录下的相应 broker 目录到需要部署的所有节点上。建议和 BE 或者 FE 目录保持同级。 @@ -114,7 +114,7 @@ broker 以插件的形式,独立于 Palo 部署。如果需要从第三方存 * 启动broker - sh bin/start_hdfs_broker.sh 启动hdfs的broker,其他类型的broker请进入相应的目录予以启动 + sh bin/start_broker.sh 启动broker。 * 添加broker diff --git a/fs_brokers/apache_hdfs_broker/bin/start_hdfs_broker.sh b/fs_brokers/apache_hdfs_broker/bin/start_broker.sh similarity index 93% rename from fs_brokers/apache_hdfs_broker/bin/start_hdfs_broker.sh rename to fs_brokers/apache_hdfs_broker/bin/start_broker.sh index d8c16bbeabcd0c..dd983c107dad7c 100755 --- a/fs_brokers/apache_hdfs_broker/bin/start_hdfs_broker.sh +++ b/fs_brokers/apache_hdfs_broker/bin/start_broker.sh @@ -21,9 +21,6 @@ curdir=`cd "$curdir"; pwd` export BROKER_HOME=`cd "$curdir/.."; pwd` export PID_DIR=`cd "$curdir"; pwd` -# -# JAVA_OPTS -# BROKER_LOG_DIR export JAVA_OPTS="-Xmx1024m -Dfile.encoding=UTF-8" export BROKER_LOG_DIR="$BROKER_HOME/log" # export JAVA_HOME="/usr/java/jdk1.8.0_131" @@ -62,12 +59,6 @@ if [ ! -d $BROKER_LOG_DIR ]; then mkdir -p $BROKER_LOG_DIR fi -if [ ! -f /bin/limit3 ]; then - LIMIT= -else - LIMIT="/bin/limit3 -c 0 -n 65536" -fi - nohup $LIMIT $JAVA $JAVA_OPTS com.baidu.palo.broker.hdfs.BrokerBootstrap "$@" >$BROKER_LOG_DIR/apache_hdfs_broker.out 2>&1 $pidfile diff --git a/fs_brokers/apache_hdfs_broker/bin/stop_hdfs_broker.sh b/fs_brokers/apache_hdfs_broker/bin/stop_broker.sh similarity index 100% rename from fs_brokers/apache_hdfs_broker/bin/stop_hdfs_broker.sh rename to fs_brokers/apache_hdfs_broker/bin/stop_broker.sh diff --git a/fs_brokers/apache_hdfs_broker/build_hdfs_broker.sh b/fs_brokers/apache_hdfs_broker/build.sh similarity index 97% rename from fs_brokers/apache_hdfs_broker/build_hdfs_broker.sh rename to fs_brokers/apache_hdfs_broker/build.sh index b1f30edb6e89bb..e4da5171429346 100755 --- a/fs_brokers/apache_hdfs_broker/build_hdfs_broker.sh +++ b/fs_brokers/apache_hdfs_broker/build.sh @@ -35,7 +35,7 @@ export BROKER_HOME=$ROOT # Every time, build deps DEPS_DIR=${BROKER_HOME}/deps -cd ${DEPS_DIR} && sh build_deps.sh +cd ${DEPS_DIR} && sh build.sh cd ${BROKER_HOME} # export all variable need by other module diff --git a/fs_brokers/apache_hdfs_broker/deps/build_deps.sh b/fs_brokers/apache_hdfs_broker/deps/build.sh similarity index 100% rename from fs_brokers/apache_hdfs_broker/deps/build_deps.sh rename to fs_brokers/apache_hdfs_broker/deps/build.sh diff --git a/fs_brokers/apache_hdfs_broker/src/com/baidu/palo/broker/hdfs/BrokerBootstrap.java b/fs_brokers/apache_hdfs_broker/src/com/baidu/palo/broker/hdfs/BrokerBootstrap.java index 2916297aa327e4..6a8a17df473afe 100644 --- a/fs_brokers/apache_hdfs_broker/src/com/baidu/palo/broker/hdfs/BrokerBootstrap.java +++ b/fs_brokers/apache_hdfs_broker/src/com/baidu/palo/broker/hdfs/BrokerBootstrap.java @@ -43,13 +43,14 @@ public static void main(String[] args) { System.setProperty("BROKER_LOG_DIR", System.getenv("BROKER_LOG_DIR")); PropertyConfigurator.configure(brokerHome + "/conf/log4j.properties"); Logger logger = Logger.getLogger(BrokerBootstrap.class); - logger.debug("starting apache hdfs broker...."); + logger.info("starting apache hdfs broker...."); new BrokerConfig().init(brokerHome + "/conf/apache_hdfs_broker.conf"); TProcessor tprocessor = new TPaloBrokerService.Processor( new HDFSBrokerServiceImpl()); ThriftServer server = new ThriftServer(BrokerConfig.broker_ipc_port, tprocessor); server.start(); + logger.info("starting apache hdfs broker....succeed"); while (true) { Thread.sleep(2000); } diff --git a/fs_brokers/baidu_bos_broker/bin/start_bos_broker.sh b/fs_brokers/baidu_bos_broker/bin/start_broker.sh similarity index 85% rename from fs_brokers/baidu_bos_broker/bin/start_bos_broker.sh rename to fs_brokers/baidu_bos_broker/bin/start_broker.sh index c1310058dc6572..d0526216e3fe23 100755 --- a/fs_brokers/baidu_bos_broker/bin/start_bos_broker.sh +++ b/fs_brokers/baidu_bos_broker/bin/start_broker.sh @@ -21,11 +21,8 @@ curdir=`cd "$curdir"; pwd` export BROKER_HOME=`cd "$curdir/.."; pwd` export PID_DIR=`cd "$curdir"; pwd` -# -# JAVA_OPTS -# LOG_DIR export JAVA_OPTS="-Xmx1024m -Dfile.encoding=UTF-8" -export LOG_DIR="$BROKER_HOME/log" +export BROKER_LOG_DIR="$BROKER_HOME/log" # export JAVA_HOME="/usr/java/jdk1.8.0_131" # java if [ "$JAVA_HOME" = "" ]; then @@ -49,7 +46,7 @@ while read line; do fi done < $BROKER_HOME/conf/baidu_bos_broker.conf -pidfile=$PID_DIR/bos_broker.pid +pidfile=$PID_DIR/baidu_bos_broker.pid if [ -f $pidfile ]; then if kill -0 `cat $pidfile` > /dev/null 2>&1; then @@ -58,16 +55,10 @@ if [ -f $pidfile ]; then fi fi -if [ ! -d $LOG_DIR ]; then - mkdir -p $LOG_DIR -fi - -if [ ! -f /bin/limit3 ]; then - LIMIT= -else - LIMIT="/bin/limit3 -c 0 -n 65536" +if [ ! -d $BROKER_LOG_DIR ]; then + mkdir -p $BROKER_LOG_DIR fi -nohup $LIMIT $JAVA $JAVA_OPTS com.baidu.palo.broker.bos.BrokerBootstrap "$@" >$LOG_DIR/bos_broker.out 2>&1 $BROKER_LOG_DIR/baidu_bos_broker.out 2>&1 $pidfile diff --git a/fs_brokers/baidu_bos_broker/bin/stop_bos_broker.sh b/fs_brokers/baidu_bos_broker/bin/stop_broker.sh similarity index 97% rename from fs_brokers/baidu_bos_broker/bin/stop_bos_broker.sh rename to fs_brokers/baidu_bos_broker/bin/stop_broker.sh index aa3535f0755aa4..1d0a486af7f931 100755 --- a/fs_brokers/baidu_bos_broker/bin/stop_bos_broker.sh +++ b/fs_brokers/baidu_bos_broker/bin/stop_broker.sh @@ -29,7 +29,7 @@ while read line; do fi done < $BROKER_HOME/conf/baidu_bos_broker.conf -pidfile=$PID_DIR/bos_broker.pid +pidfile=$PID_DIR/baidu_bos_broker.pid if [ -f $pidfile ]; then pid=`cat $pidfile` diff --git a/fs_brokers/baidu_bos_broker/build_bos_broker.sh b/fs_brokers/baidu_bos_broker/build.sh similarity index 96% rename from fs_brokers/baidu_bos_broker/build_bos_broker.sh rename to fs_brokers/baidu_bos_broker/build.sh index 834777690bba1e..e4da5171429346 100755 --- a/fs_brokers/baidu_bos_broker/build_bos_broker.sh +++ b/fs_brokers/baidu_bos_broker/build.sh @@ -35,12 +35,12 @@ export BROKER_HOME=$ROOT # Every time, build deps DEPS_DIR=${BROKER_HOME}/deps -cd ${DEPS_DIR} && sh build_deps.sh +cd ${DEPS_DIR} && sh build.sh cd ${BROKER_HOME} # export all variable need by other module export PATH=${DEPS_DIR}/bin:$PATH ANT_HOME=${DEPS_DIR}/ant export PATH=${ANT_HOME}/bin:$PATH -ant release +ant output exit diff --git a/fs_brokers/baidu_bos_broker/deps/baidu_bos_broker_java_libraries.tar.gz b/fs_brokers/baidu_bos_broker/deps/baidu_bos_broker_java_libraries.tar.gz new file mode 100644 index 00000000000000..1e013907f55109 Binary files /dev/null and b/fs_brokers/baidu_bos_broker/deps/baidu_bos_broker_java_libraries.tar.gz differ diff --git a/fs_brokers/baidu_bos_broker/deps/build_deps.sh b/fs_brokers/baidu_bos_broker/deps/build.sh similarity index 79% rename from fs_brokers/baidu_bos_broker/deps/build_deps.sh rename to fs_brokers/baidu_bos_broker/deps/build.sh index 537b68815c467e..81f008ed921ecc 100755 --- a/fs_brokers/baidu_bos_broker/deps/build_deps.sh +++ b/fs_brokers/baidu_bos_broker/deps/build.sh @@ -23,7 +23,7 @@ DEPSDIR=`dirname "$0"` DEPSDIR=`cd ${DEPSDIR}; pwd` CURDIR=`pwd` -if [ ! -f ${DEPSDIR}/bin/thrift ] +if [ ! -f ${DEPSDIR}/bin/bce-java-sdk-0.9.1-internal.jar ] then echo "***********************************" echo " Downloading dependency libraries " @@ -31,9 +31,16 @@ then cd ${DEPSDIR} # Check out depends # extract archive - tar xzf baidu_bos_broker_deps.tar.gz + tar xzf baidu_bos_broker_java_libraries.tar.gz echo "Unpacking dependency libraries...Done " else echo "Depends already exists." fi + +if [ ! -f bin/thrift ];then + echo "thrift is not found." + echo "You need to copy thrift binary file from 'thirdparty/installed/bin/thrift' to $CURDIR" + exit 1 +fi + cd ${CURDIR}