From 92d69ff40f0062c497678b0a5069c63e243c46a6 Mon Sep 17 00:00:00 2001
From: admondguo <admondguo@163.com>
Date: Mon, 25 Apr 2022 18:09:45 +0800
Subject: [PATCH 1/5] add example for all-in-one & server mode.

---
 conf/byzer.properties                         | 12 ++++----
 conf/byzer.properties.all-in-one.example      | 13 +++++++++
 conf/byzer.properties.server.example          | 18 ++++++++++++
 dev/bootstrap.sh                              | 28 +++++++++++++------
 .../java/tech/mlsql/tool/ByzerConfigCLI.java  | 17 +++++++++--
 5 files changed, 70 insertions(+), 18 deletions(-)
 create mode 100644 conf/byzer.properties.all-in-one.example
 create mode 100644 conf/byzer.properties.server.example

diff --git a/conf/byzer.properties b/conf/byzer.properties
index 4bfabf403..42e019ded 100644
--- a/conf/byzer.properties
+++ b/conf/byzer.properties
@@ -25,13 +25,13 @@ byzer.server.mode=server
 
 # spark properties
 spark.master=local[*]
-spark.driver.memory=2g
-spark.sql.hive.thriftServer.singleSession=true
-spark.kryoserializer.buffer=256k
-spark.kryoserializer.buffer.max=1024m
+#spark.driver.memory=2g
+#spark.sql.hive.thriftServer.singleSession=true
+#spark.kryoserializer.buffer=256k
+#spark.kryoserializer.buffer.max=1024m
 #spark.sql.shuffle.partitions=1
-spark.serializer=org.apache.spark.serializer.KryoSerializer
-spark.scheduler.mode=FAIR
+#spark.serializer=org.apache.spark.serializer.KryoSerializer
+#spark.scheduler.mode=FAIR
 #spark.hadoop.mapreduce.job.run-local=true
 #spark.driver.maxResultSize=2g
 #spark.sql.codegen.wholeStage=true
diff --git a/conf/byzer.properties.all-in-one.example b/conf/byzer.properties.all-in-one.example
new file mode 100644
index 000000000..a0cf9fe77
--- /dev/null
+++ b/conf/byzer.properties.all-in-one.example
@@ -0,0 +1,13 @@
+# override example for all-in-one package
+
+streaming.master=local[*]
+streaming.name=Byzer-lang-desktop
+streaming.rest=true
+streaming.thrift=false
+streaming.platform=spark
+streaming.spark.service=true
+streaming.job.cancel=true
+streaming.datalake.path=./data/
+streaming.driver.port=9003
+streaming.enableHiveSupport=false
+streaming.plugin.clzznames=tech.mlsql.plugins.ds.MLSQLExcelApp,tech.mlsql.plugins.assert.app.MLSQLAssert,tech.mlsql.plugins.shell.app.MLSQLShell,tech.mlsql.plugins.ext.ets.app.MLSQLETApp,tech.mlsql.plugins.mllib.app.MLSQLMllib
diff --git a/conf/byzer.properties.server.example b/conf/byzer.properties.server.example
new file mode 100644
index 000000000..b60eb366d
--- /dev/null
+++ b/conf/byzer.properties.server.example
@@ -0,0 +1,18 @@
+# override example for server package
+
+streaming.master=local[*]
+streaming.name=Byzer-lang-server
+streaming.rest=true
+streaming.thrift=false
+streaming.platform=spark
+streaming.spark.service=true
+streaming.driver.port=9003
+streaming.enableHiveSupport=false
+
+spark.master=local[*]
+spark.driver.memory=2g
+spark.scheduler.mode=FAIR
+spark.sql.hive.thriftServer.singleSession=true
+spark.kryoserializer.buffer=256k
+spark.kryoserializer.buffer.max=1024m
+spark.serializer=org.apache.spark.serializer.KryoSerializer
\ No newline at end of file
diff --git a/dev/bootstrap.sh b/dev/bootstrap.sh
index 263e92774..c9644b8f3 100755
--- a/dev/bootstrap.sh
+++ b/dev/bootstrap.sh
@@ -88,13 +88,14 @@ function prepareProp() {
 
     BYZER_PROP=$($BYZER_HOME/bin/get-properties.sh -byzer)
     SPARK_PROP=$($BYZER_HOME/bin/get-properties.sh -spark)
+    ALL_PROP=$(BYZER_HOME/bin/get-properties.sh -args)
 }
 
 function start(){
     clearRedundantProcess
 
     # check $BYZER_HOME
-    [[ -z ${BYZER_HOME} ]] && quit "{BYZER_HOME} is not set, exit" 
+    [[ -z ${BYZER_HOME} ]] && quit "{BYZER_HOME} is not set, exit"
     if [ -f "${BYZER_HOME}/pid" ]; then
         PID=$(cat ${BYZER_HOME}/pid)
         if ps -p $PID > /dev/null; then
@@ -105,7 +106,7 @@ function start(){
     # check $SPARK_HOME
     if [ $BYZER_SERVER_MODE == "server" ]; then
         # only in server mode need check spark home
-        [[ -z ${SPARK_HOME} ]]  && quit "{SPARK_HOME} is not set, exit" 
+        [[ -z ${SPARK_HOME} ]]  && quit "{SPARK_HOME} is not set, exit"
     fi
 
     ${BYZER_HOME}/bin/check-env.sh || exit 1
@@ -129,13 +130,22 @@ function start(){
     echo "${EXT_JARS}"
 
     cd $BYZER_HOME/
-    nohup $SPARK_HOME/bin/spark-submit --class streaming.core.StreamingApp \
-        --jars ${JARS} \
-        --conf "spark.driver.extraClassPath=${EXT_JARS}" \
-        --driver-java-options "-Dlog4j.configuration=${BYZER_LOG_PATH}" \
-        $SPARK_PROP \
-        $MAIN_JAR_PATH  \
-        $BYZER_PROP >> ${BYZER_HOME}/logs/byzer.out & echo $! >> ${BYZER_HOME}/pid
+
+    if [[ $BYZER_SERVER_MODE = "all-in-one" ]]; then
+        nohup ${BYZER_HOME}/jdk8/bin/java -cp ${BYZER_HOME}/main/${MAIN_JAR}:${BYZER_HOME}/spark/*:${BYZER_HOME}/libs/*:${BYZER_HOME}/plugin/* \
+            tech.mlsql.example.app.LocalSparkServiceApp \
+            $ALL_PROP >> ${BYZER_HOME}/logs/byzer.out \
+            & echo $! >> ${BYZER_HOME}/pid
+
+    elif [[ $BYZER_SERVER_MODE = "server" ]]; then
+        nohup $SPARK_HOME/bin/spark-submit --class streaming.core.StreamingApp \
+            --jars ${JARS} \
+            --conf "spark.driver.extraClassPath=${EXT_JARS}" \
+            --driver-java-options "-Dlog4j.configuration=${BYZER_LOG_PATH}" \
+            $SPARK_PROP \
+            $MAIN_JAR_PATH  \
+            $BYZER_PROP >> ${BYZER_HOME}/logs/byzer.out & echo $! >> ${BYZER_HOME}/pid
+    fi
 
     sleep 3
     clearRedundantProcess
diff --git a/streamingpro-commons/src/main/java/tech/mlsql/tool/ByzerConfigCLI.java b/streamingpro-commons/src/main/java/tech/mlsql/tool/ByzerConfigCLI.java
index 5920b4bf2..340f1190c 100644
--- a/streamingpro-commons/src/main/java/tech/mlsql/tool/ByzerConfigCLI.java
+++ b/streamingpro-commons/src/main/java/tech/mlsql/tool/ByzerConfigCLI.java
@@ -30,6 +30,8 @@ public class ByzerConfigCLI {
 
     private final static String BYZER_CONF_TEMP = "-%s %s";
 
+    private final static String ARGS_CONF_TEMP = "-%s %s ";
+
     public static void main(String[] args) {
         execute(args);
         Unsafe.systemExit(0);
@@ -51,7 +53,7 @@ public static void execute(String[] args) {
 
         String key = args[0].trim();
         if (key.equals("-byzer")) {
-            // get all properties
+            // get byzer properties
             for (Map.Entry<Object, Object> entry : config.entrySet()) {
                 String entryKey = (String) entry.getKey();
                 if (entryKey.startsWith("streaming") || entryKey.startsWith("spark.mlsql")) {
@@ -60,7 +62,7 @@ public static void execute(String[] args) {
                 }
             }
         } else if (key.equals("-spark")) {
-            // get all properties
+            // get spark properties
             for (Map.Entry<Object, Object> entry : config.entrySet()) {
                 String entryKey = (String) entry.getKey();
                 if (entryKey.startsWith("spark") && !entryKey.startsWith("spark.mlsql")) {
@@ -68,7 +70,16 @@ public static void execute(String[] args) {
                     System.out.println(prop);
                 }
             }
-        } else if (!key.endsWith(".")) {
+        } else if (key.equals("-args")) {
+            // get all properties
+            StringBuffer prop = new StringBuffer("");
+            for (Map.Entry<Object, Object> entry : config.entrySet()) {
+                String entryKey = (String) entry.getKey();
+                prop.append(String.format(ARGS_CONF_TEMP, entryKey, entry.getValue()));
+            }
+            System.out.println(prop);
+        }
+        else if (!key.endsWith(".")) {
             String value = config.getProperty(key);
             if (value == null) {
                 value = "";

From 830e8dd8ee47779d4e66bc926f3fa8c278ca803f Mon Sep 17 00:00:00 2001
From: admondguo <admondguo@163.com>
Date: Mon, 25 Apr 2022 21:56:18 +0800
Subject: [PATCH 2/5] add assembly include & fix properties

---
 conf/byzer.properties                                | 12 ++++++------
 conf/byzer.properties.server.example                 | 10 +---------
 streamingpro-assembly/src/main/assembly/assembly.xml |  2 ++
 3 files changed, 9 insertions(+), 15 deletions(-)

diff --git a/conf/byzer.properties b/conf/byzer.properties
index 42e019ded..4bfabf403 100644
--- a/conf/byzer.properties
+++ b/conf/byzer.properties
@@ -25,13 +25,13 @@ byzer.server.mode=server
 
 # spark properties
 spark.master=local[*]
-#spark.driver.memory=2g
-#spark.sql.hive.thriftServer.singleSession=true
-#spark.kryoserializer.buffer=256k
-#spark.kryoserializer.buffer.max=1024m
+spark.driver.memory=2g
+spark.sql.hive.thriftServer.singleSession=true
+spark.kryoserializer.buffer=256k
+spark.kryoserializer.buffer.max=1024m
 #spark.sql.shuffle.partitions=1
-#spark.serializer=org.apache.spark.serializer.KryoSerializer
-#spark.scheduler.mode=FAIR
+spark.serializer=org.apache.spark.serializer.KryoSerializer
+spark.scheduler.mode=FAIR
 #spark.hadoop.mapreduce.job.run-local=true
 #spark.driver.maxResultSize=2g
 #spark.sql.codegen.wholeStage=true
diff --git a/conf/byzer.properties.server.example b/conf/byzer.properties.server.example
index b60eb366d..b32bf368b 100644
--- a/conf/byzer.properties.server.example
+++ b/conf/byzer.properties.server.example
@@ -7,12 +7,4 @@ streaming.thrift=false
 streaming.platform=spark
 streaming.spark.service=true
 streaming.driver.port=9003
-streaming.enableHiveSupport=false
-
-spark.master=local[*]
-spark.driver.memory=2g
-spark.scheduler.mode=FAIR
-spark.sql.hive.thriftServer.singleSession=true
-spark.kryoserializer.buffer=256k
-spark.kryoserializer.buffer.max=1024m
-spark.serializer=org.apache.spark.serializer.KryoSerializer
\ No newline at end of file
+streaming.enableHiveSupport=false
\ No newline at end of file
diff --git a/streamingpro-assembly/src/main/assembly/assembly.xml b/streamingpro-assembly/src/main/assembly/assembly.xml
index c3b41b6dd..47ec61403 100644
--- a/streamingpro-assembly/src/main/assembly/assembly.xml
+++ b/streamingpro-assembly/src/main/assembly/assembly.xml
@@ -51,6 +51,8 @@
             <includes>
                 <include>byzer.properties</include>
                 <include>byzer.properties.override</include>
+                <include>byzer.properties.all-in-one.example</include>
+                <include>byzer.properties.server.example</include>
                 <include>byzer-server-log4j.properties</include>
                 <include>byzer-tools-log4j.properties</include>
             </includes>

From 8da60eee63225fed35018443b1a532d43b09f30e Mon Sep 17 00:00:00 2001
From: admondguo <admondguo@163.com>
Date: Mon, 25 Apr 2022 22:15:42 +0800
Subject: [PATCH 3/5] fix properties example

---
 conf/byzer.properties.all-in-one.example | 2 ++
 conf/byzer.properties.server.example     | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/conf/byzer.properties.all-in-one.example b/conf/byzer.properties.all-in-one.example
index a0cf9fe77..eb69dcf81 100644
--- a/conf/byzer.properties.all-in-one.example
+++ b/conf/byzer.properties.all-in-one.example
@@ -1,5 +1,7 @@
 # override example for all-in-one package
 
+byzer.server.mode=all-in-one
+
 streaming.master=local[*]
 streaming.name=Byzer-lang-desktop
 streaming.rest=true
diff --git a/conf/byzer.properties.server.example b/conf/byzer.properties.server.example
index b32bf368b..69609246c 100644
--- a/conf/byzer.properties.server.example
+++ b/conf/byzer.properties.server.example
@@ -1,5 +1,7 @@
 # override example for server package
 
+byzer.server.mode=server
+
 streaming.master=local[*]
 streaming.name=Byzer-lang-server
 streaming.rest=true

From a111a0c44c1197dcde4e831157ff52438b6ff6c9 Mon Sep 17 00:00:00 2001
From: Zhengshuai PENG <zhengshuai.peng@outlook.com>
Date: Mon, 25 Apr 2022 22:42:20 +0800
Subject: [PATCH 4/5] minor fix

---
 dev/bootstrap.sh | 30 ++++++++++++++++++++----------
 1 file changed, 20 insertions(+), 10 deletions(-)

diff --git a/dev/bootstrap.sh b/dev/bootstrap.sh
index c9644b8f3..804ae4126 100755
--- a/dev/bootstrap.sh
+++ b/dev/bootstrap.sh
@@ -88,7 +88,7 @@ function prepareProp() {
 
     BYZER_PROP=$($BYZER_HOME/bin/get-properties.sh -byzer)
     SPARK_PROP=$($BYZER_HOME/bin/get-properties.sh -spark)
-    ALL_PROP=$(BYZER_HOME/bin/get-properties.sh -args)
+    ALL_PROP=$($BYZER_HOME/bin/get-properties.sh -args)
 }
 
 function start(){
@@ -120,24 +120,32 @@ function start(){
     prepareProp
 
     echo "Starting Byzer engine in ${BYZER_SERVER_MODE} mode..."
-    echo "[Spark config]"
-    echo "$SPARK_PROP"
-
-    echo "[Byzer config]"
-    echo "${BYZER_PROP}"
-
-    echo "[Extra config]"
-    echo "${EXT_JARS}"
 
     cd $BYZER_HOME/
 
     if [[ $BYZER_SERVER_MODE = "all-in-one" ]]; then
+        echo ""
+        echo "[All Config]"
+        echo "${ALL_PROP}"
+        echo ""
+        
         nohup ${BYZER_HOME}/jdk8/bin/java -cp ${BYZER_HOME}/main/${MAIN_JAR}:${BYZER_HOME}/spark/*:${BYZER_HOME}/libs/*:${BYZER_HOME}/plugin/* \
             tech.mlsql.example.app.LocalSparkServiceApp \
             $ALL_PROP >> ${BYZER_HOME}/logs/byzer.out \
             & echo $! >> ${BYZER_HOME}/pid
 
     elif [[ $BYZER_SERVER_MODE = "server" ]]; then
+        echo ""
+        echo "[Spark Config]"
+        echo "$SPARK_PROP"
+
+        echo "[Byzer Config]"
+        echo "${BYZER_PROP}"
+
+        echo "[Extra Config]"
+        echo "${EXT_JARS}"
+        echo ""
+
         nohup $SPARK_HOME/bin/spark-submit --class streaming.core.StreamingApp \
             --jars ${JARS} \
             --conf "spark.driver.extraClassPath=${EXT_JARS}" \
@@ -156,7 +164,9 @@ function start(){
     CUR_DATE=$(date "+%Y-%m-%d %H:%M:%S")
     echo $CUR_DATE" new Byzer engine process pid is "$PID >> ${BYZER_HOME}/logs/byzer-lang.log
 
-    echo "Byzer engine is starting. It may take a while. For status, please visit http://$BYZER_IP:$BYZER_LANG_PORT."
+    echo ""
+    echo $(setColor 33 "Byzer engine is starting. It may take a while. For status, please visit http://$BYZER_IP:$BYZER_LANG_PORT.")
+    echo ""
     echo "You may also check status via: PID:$(cat ${BYZER_HOME}/pid), or Log: ${BYZER_HOME}/logs/byzer-lang.log."
     recordStartOrStop "start success" "${START_TIME}"
 }

From 27ecb90f10a7532c19cab2259eb189945c7ab0d5 Mon Sep 17 00:00:00 2001
From: Zhengshuai PENG <zhengshuai.peng@outlook.com>
Date: Mon, 25 Apr 2022 23:23:03 +0800
Subject: [PATCH 5/5] format minor fix

---
 dev/bootstrap.sh                                           | 7 ++++---
 .../src/main/java/tech/mlsql/tool/ByzerConfigCLI.java      | 2 +-
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/dev/bootstrap.sh b/dev/bootstrap.sh
index 804ae4126..cbd95efd0 100755
--- a/dev/bootstrap.sh
+++ b/dev/bootstrap.sh
@@ -131,8 +131,8 @@ function start(){
         
         nohup ${BYZER_HOME}/jdk8/bin/java -cp ${BYZER_HOME}/main/${MAIN_JAR}:${BYZER_HOME}/spark/*:${BYZER_HOME}/libs/*:${BYZER_HOME}/plugin/* \
             tech.mlsql.example.app.LocalSparkServiceApp \
-            $ALL_PROP >> ${BYZER_HOME}/logs/byzer.out \
-            & echo $! >> ${BYZER_HOME}/pid
+            $ALL_PROP >> ${BYZER_HOME}/logs/byzer.out &
+        echo $! >> ${BYZER_HOME}/pid
 
     elif [[ $BYZER_SERVER_MODE = "server" ]]; then
         echo ""
@@ -152,7 +152,8 @@ function start(){
             --driver-java-options "-Dlog4j.configuration=${BYZER_LOG_PATH}" \
             $SPARK_PROP \
             $MAIN_JAR_PATH  \
-            $BYZER_PROP >> ${BYZER_HOME}/logs/byzer.out & echo $! >> ${BYZER_HOME}/pid
+            $BYZER_PROP >> ${BYZER_HOME}/logs/byzer.out & 
+        echo $! >> ${BYZER_HOME}/pid
     fi
 
     sleep 3
diff --git a/streamingpro-commons/src/main/java/tech/mlsql/tool/ByzerConfigCLI.java b/streamingpro-commons/src/main/java/tech/mlsql/tool/ByzerConfigCLI.java
index 340f1190c..7857d8b8f 100644
--- a/streamingpro-commons/src/main/java/tech/mlsql/tool/ByzerConfigCLI.java
+++ b/streamingpro-commons/src/main/java/tech/mlsql/tool/ByzerConfigCLI.java
@@ -70,7 +70,7 @@ public static void execute(String[] args) {
                     System.out.println(prop);
                 }
             }
-        } else if (key.equals("-args")) {
+        } else if ("-args".equals(key)) {
             // get all properties
             StringBuffer prop = new StringBuffer("");
             for (Map.Entry<Object, Object> entry : config.entrySet()) {