统一安装及运行时的env.sh变量读取

assembly-package/sbin/common.sh

@@ -14,6 +14,5 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
 declare -A MODULE_MAIN_CLASS
-MODULE_MAIN_CLASS["dss-exchangis-main-server-dev"]="com.webank.wedatasphere.exchangis.server.boot.ExchangisServerApplication"
+MODULE_MAIN_CLASS["${MODULE_DEFAULT_PREFIX}server${MODULE_DEFAULT_SUFFIX}"]="com.webank.wedatasphere.exchangis.server.boot.ExchangisServerApplication"

assembly-package/sbin/daemon.sh

@@ -48,7 +48,7 @@ restart(){
 COMMAND=$1
 case $COMMAND in
   start|stop|restart)
-    load_env_definitions ${ENV_FILE}
+    load_env_definitions
     if [[ ! -z $2 ]]; then
       SERVICE_NAME=${MODULE_DEFAULT_PREFIX}$2${MODULE_DEFAULT_SUFFIX}
       MAIN_CLASS=${MODULE_MAIN_CLASS[${SERVICE_NAME}]}

assembly-package/sbin/env.properties → assembly-package/sbin/env.sh

@@ -1,4 +1,4 @@
 EXCHANGIS_CONF_PATH=/appcom/config/exchangis-config/background
 EXCHANGIS_LOG_PATH=/appcom/logs/exchangis/background
 MODULE_DEFAULT_PREFIX="dss-exchangis-main-"
-MODULE_DEFAULT_SUFFIX="-dev"
+MODULE_DEFAULT_SUFFIX="-dev"

assembly-package/sbin/install.sh

@@ -23,9 +23,11 @@ SOURCE_ROOT=${workDir}
 #load config
 source ${SOURCE_ROOT}/config/config.sh
 source ${SOURCE_ROOT}/config/db.sh
+source ${SOURCE_ROOT}/sbin/env.sh
 DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 SHELL_LOG="${DIR}/console.out"   #console.out是什么文件？
 export SQL_SOURCE_PATH="${DIR}/../db/exchangis_ddl.sql"
+export SQL_DMLSOURCE_PATH="${DIR}/../db/exchangis_dml.sql"
 PACKAGE_DIR="${DIR}/../packages"
 # Home Path
 EXCHNGIS_HOME_PATH="${DIR}/../"
@@ -125,24 +127,24 @@ interact_echo(){
 
 # Initalize database
 init_database(){
-    BOOTSTRAP_PROP_FILE="${CONF_PATH}/dss-exchangis-server.properties"
+    BOOTSTRAP_PROP_FILE="${CONF_PATH}/${MODULE_DEFAULT_PREFIX}server${MODULE_DEFAULT_SUFFIX}.properties"
     if [ "x${SQL_SOURCE_PATH}" != "x" ] && [ -f "${SQL_SOURCE_PATH}" ]; then
         `mysql --version >/dev/null 2>&1`
         DATASOURCE_URL="jdbc:mysql:\/\/${MYSQL_HOST}:${MYSQL_PORT}\/${DATABASE}\?useSSL=false\&characterEncoding=UTF-8\&allowMultiQueries=true"
         sed -ri "s![#]?(wds.linkis.server.mybatis.datasource.username=)\S*!\1${MYSQL_USERNAME}!g" ${BOOTSTRAP_PROP_FILE}
         sed -ri "s![#]?(wds.linkis.server.mybatis.datasource.password=)\S*!\1${MYSQL_PASSWORD}!g" ${BOOTSTRAP_PROP_FILE}
         sed -ri "s![#]?(wds.linkis.server.mybatis.datasource.url=)\S*!\1${DATASOURCE_URL}!g" ${BOOTSTRAP_PROP_FILE}
-        interact_echo "Do you want to initalize database with sql: [${SQL_SOURCE_PATH}]?"
+        interact_echo "Do you want to initalize database with sql: [${SQL_SOURCE_PATH}]  [${SQL_DMLSOURCE_PATH}] ?"
         if [ $? == 0 ]; then
           LOG INFO "\033[1m Scan out mysql command, so begin to initalize the database\033[0m"
           mysql -h ${MYSQL_HOST} -P ${MYSQL_PORT} -u ${MYSQL_USERNAME} -p${MYSQL_PASSWORD}  --default-character-set=utf8 -e \
-          "CREATE DATABASE IF NOT EXISTS ${DATABASE}; USE ${DATABASE}; source ${SQL_SOURCE_PATH};"
+          "CREATE DATABASE IF NOT EXISTS ${DATABASE}; USE ${DATABASE}; source ${SQL_SOURCE_PATH};source ${SQL_DMLSOURCE_PATH};"
         fi
     fi
 }
 
 init_properties(){
-    BOOTSTRAP_PROP_FILE="${CONF_PATH}/dss-exchangis-server.properties"
+    BOOTSTRAP_PROP_FILE="${CONF_PATH}/${MODULE_DEFAULT_PREFIX}server${MODULE_DEFAULT_SUFFIX}.properties"
     APPLICATION_YML="${CONF_PATH}/application-exchangis.yml"
     LINKIS_GATEWAY_URL="http:\/\/${LINKIS_GATEWAY_HOST}:${LINKIS_GATEWAY_PORT}\/"
     if [ "x${LINKIS_SERVER_URL}" == "x" ]; then
@@ -158,6 +160,8 @@ init_properties(){
 install_modules(){
   LOG INFO "\033[1m ####### Start To Install project ######\033[0m"
   echo ""
+    LOG INFO "\033[1m Prepare properties file ${MODULE_DEFAULT_PREFIX}server${MODULE_DEFAULT_SUFFIX}.properties ......\033[0m"
+    cp -f ${EXCHNGIS_HOME_PATH}/config/dss-exchangis-server.properties ${EXCHNGIS_HOME_PATH}/config/${MODULE_DEFAULT_PREFIX}server${MODULE_DEFAULT_SUFFIX}.properties
     if [ ${FORCE_INSTALL} == false ]; then
         LOG INFO "\033[1m Install project ......\033[0m"
         init_database

assembly-package/sbin/launcher.sh

@@ -17,7 +17,8 @@
 # Launcher for modules, provided start/stop functions
 
 DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-ENV_FILE="${DIR}/env.properties"
+#ENV_FILE="${DIR}/env.properties"
+source ${DIR}/env.sh
 SHELL_LOG="${DIR}/command.log"
 USER_DIR="${DIR}/../"
 EXCHANGIS_LIB_PATH="${DIR}/../lib"
@@ -64,20 +65,6 @@ EOF
 
 # load environment definition
 load_env_definitions(){
-   if [[ -f "$1" ]]; then
-      LOG INFO "load environment properties"
-      while read line
-      do
-        if [[ ! -z $(echo "${line}" | grep "=") ]]; then
-          local key=${line%%=*}
-          local value=${line##*=}
-          local key1=$(echo ${key} | tr '.' '_')
-          if [[ -z $(echo "${key1}" | grep -P '\s*#+.*') ]]; then
-            eval "${key1}=${value}"
-          fi
-        fi
-      done < "${ENV_FILE}"
-   fi
    if [[ "x${JAVA_HOME}" != "x" ]]; then
       JPS=${JAVA_HOME}/bin/jps
    else

db/exchangis_dml.sql

@@ -64,7 +64,7 @@ INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_directio
 
 -- engine_settings records
 INSERT INTO `exchangis_engine_settings` (id, engine_name, engine_desc, engine_settings_value, engine_direction, res_loader_class, res_uploader_class, modify_time) VALUES
-(1, 'datax', 'datax sync engine', '{}', 'mysql->hive,hive->mysql,mysql->oracle,oracle->mysql,oracle->hive,hive->oracle,mongodb->hive,hive->mongodb,mysql->elasticsearch,oracle->elasticsearch,mongodb->elasticsearch,mysql->mongodb,mongodb->mysql,oracle->mongodb,mongodb->oracle', 'com.webank.wedatasphere.exchangis.engine.resource.loader.datax.DataxEngineResourceLoader', NULL, NULL, '2022-08-09 18:20:51.0'),
+(1, 'datax', 'datax sync engine', '{}', 'mysql->hive,hive->mysql,mysql->oracle,oracle->mysql,oracle->hive,hive->oracle,mongodb->hive,hive->mongodb,mysql->elasticsearch,oracle->elasticsearch,mongodb->elasticsearch,mysql->mongodb,mongodb->mysql,oracle->mongodb,mongodb->oracle', 'com.webank.wedatasphere.exchangis.engine.resource.loader.datax.DataxEngineResourceLoader', NULL, NULL),
 (2, 'sqoop', 'hadoop tool', '{}', 'mysql->hive,hive->mysql', '', NULL, NULL);
 
 -- exchangis_job_transform_rule records

exchangis-datasource/exchangis-datasource-service/src/main/java/com/webank/wedatasphere/exchangis/datasource/Utils/RSAUtil.java

@@ -1,8 +1,7 @@
 package com.webank.wedatasphere.exchangis.datasource.Utils;
 
+import org.apache.commons.codec.binary.Base64;
 import org.apache.linkis.common.conf.CommonVars;
-import sun.misc.BASE64Decoder;
-import sun.misc.BASE64Encoder;
 
 import javax.crypto.Cipher;
 import java.io.IOException;
@@ -29,21 +28,21 @@ public static KeyPair getKeyPair() throws Exception {
     }
 
     //获取公钥(Base64编码)
-    public static String getPublicKey(KeyPair keyPair){
+    public static String getPublicKey(KeyPair keyPair) {
         PublicKey publicKey = keyPair.getPublic();
         byte[] bytes = publicKey.getEncoded();
         return byte2Base64(bytes);
     }
 
     //获取私钥(Base64编码)
-    public static String getPrivateKey(KeyPair keyPair){
+    public static String getPrivateKey(KeyPair keyPair) {
         PrivateKey privateKey = keyPair.getPrivate();
         byte[] bytes = privateKey.getEncoded();
         return byte2Base64(bytes);
     }
 
     //将Base64编码后的公钥转换成PublicKey对象
-    public static PublicKey string2PublicKey(String pubStr) throws Exception{
+    public static PublicKey string2PublicKey(String pubStr) throws Exception {
         byte[] keyBytes = base642Byte(pubStr);
         X509EncodedKeySpec keySpec = new X509EncodedKeySpec(keyBytes);
         KeyFactory keyFactory = KeyFactory.getInstance("RSA");
@@ -52,7 +51,7 @@ public static PublicKey string2PublicKey(String pubStr) throws Exception{
     }
 
     //将Base64编码后的私钥转换成PrivateKey对象
-    public static PrivateKey string2PrivateKey(String priStr) throws Exception{
+    public static PrivateKey string2PrivateKey(String priStr) throws Exception {
         byte[] keyBytes = base642Byte(priStr);
         PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(keyBytes);
         KeyFactory keyFactory = KeyFactory.getInstance("RSA");
@@ -61,30 +60,32 @@ public static PrivateKey string2PrivateKey(String priStr) throws Exception{
     }
 
     //公钥加密
-    public static byte[] publicEncrypt(byte[] content, PublicKey publicKey) throws Exception{
+    public static byte[] publicEncrypt(byte[] content, PublicKey publicKey) throws Exception {
         Cipher cipher = Cipher.getInstance("RSA");
         cipher.init(Cipher.ENCRYPT_MODE, publicKey);
         byte[] bytes = cipher.doFinal(content);
         return bytes;
     }
 
     //私钥解密
-    public static byte[] privateDecrypt(byte[] content, PrivateKey privateKey) throws Exception{
+    public static byte[] privateDecrypt(byte[] content, PrivateKey privateKey) throws Exception {
         Cipher cipher = Cipher.getInstance("RSA");
         cipher.init(Cipher.DECRYPT_MODE, privateKey);
         byte[] bytes = cipher.doFinal(content);
         return bytes;
     }
 
     //字节数组转Base64编码
-    public static String byte2Base64(byte[] bytes){
-        BASE64Encoder encoder = new BASE64Encoder();
-        return encoder.encode(bytes);
+    public static String byte2Base64(byte[] bytes) {
+//        BASE64Encoder encoder = new BASE64Encoder();
+//        return encoder.encode(bytes);
+        return Base64.encodeBase64String(bytes);
     }
 
     //Base64编码转字节数组
     public static byte[] base642Byte(String base64Key) throws IOException {
-        BASE64Decoder decoder = new BASE64Decoder();
-        return decoder.decodeBuffer(base64Key);
+//        BASE64Decoder decoder = new BASE64Decoder();
+//        return decoder.decodeBuffer(base64Key);
+        return Base64.decodeBase64(base64Key);
     }
 }

exchangis-engines/engines/datax/datax-hdfsreader/pom.xml

@@ -96,6 +96,59 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>org.apache.hive</groupId>
+            <artifactId>hive-metastore</artifactId>
+            <version>${hive.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>org.apache.hive</groupId>
+                    <artifactId>hive-serde</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hive</groupId>
+                    <artifactId>hive-shims</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.thrift</groupId>
+                    <artifactId>libfb303</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.thrift</groupId>
+                    <artifactId>libthrift</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.mortbay.jetty</groupId>
+                    <artifactId>servlet-api</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.google.guava</groupId>
+                    <artifactId>guava</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-api</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.slf4j</groupId>
+                    <artifactId>slf4j-log4j12</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.apache.hbase</groupId>
+                    <artifactId>hbase-client</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>co.cask.tephra</groupId>
+                    <artifactId>*</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.paimon</groupId>
+            <artifactId>paimon-hive-connector-3.1</artifactId>
+            <version>${paimon.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>xalan</groupId>
             <artifactId>xalan</artifactId>

exchangis-engines/engines/datax/datax-hdfswriter/pom.xml

@@ -105,6 +105,11 @@
                 </exclusion>
             </exclusions>
         </dependency>
+        <dependency>
+            <groupId>org.apache.paimon</groupId>
+            <artifactId>paimon-hive-connector-3.1</artifactId>
+            <version>${paimon.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.hive</groupId>
             <artifactId>hive-common</artifactId>

exchangis-server/pom.xml

@@ -31,10 +31,6 @@
                     <groupId>org.apache.linkis</groupId>
                     <artifactId>linkis-module</artifactId>
                 </exclusion>
-                <exclusion>
-                    <artifactId>spring-jdbc</artifactId>
-                    <groupId>org.springframework</groupId>
-                </exclusion>
             </exclusions>
         </dependency>
         <dependency>

pom.xml

@@ -42,10 +42,12 @@
 		<linkis.version>1.4.0</linkis.version>
 		<linkis.datasource.version>1.4.0</linkis.datasource.version>
 		<streamis.version>0.1.0-SNAPSHOT</streamis.version>
-		<scala.version>2.12.12</scala.version>
+		<scala.version>2.12.18</scala.version>
 		<scala-maven-plugin.version>4.7.1</scala-maven-plugin.version>
 		<jdk.compile.version>1.8</jdk.compile.version>
-		<maven.version>3.3.3</maven.version>
+		<maven.compiler.source>${jdk.compile.version}</maven.compiler.source>
+		<maven.compiler.target>${jdk.compile.version}</maven.compiler.target>
+		<maven.version>3.9.8</maven.version>
 		<gson.version>2.8.5</gson.version>
 		<fasterxml.jackson.version>2.11.3</fasterxml.jackson.version>
 		<org.codehaus.jackson.version>1.9.13</org.codehaus.jackson.version>
@@ -66,6 +68,7 @@
 		<aspectj.version>1.9.5</aspectj.version>
 		<xstream.version>1.4.19</xstream.version>
 		<jobmanager.version>0.1.0-SNAPSHOT</jobmanager.version>
+		<paimon.version>0.8.1</paimon.version>
 	</properties>
 
 	<modules>
@@ -182,6 +185,16 @@
 	<build>
 		<pluginManagement>
 			<plugins>
+				<plugin>
+					<groupId>org.apache.maven.plugins</groupId>
+					<artifactId>maven-compiler-plugin</artifactId>
+					<version>3.1</version>
+					<configuration>
+						<source>${jdk.compile.version}</source>
+						<target>${jdk.compile.version}</target>
+						<encoding>${project.build.sourceEncoding}</encoding>
+					</configuration>
+				</plugin>
 				<plugin>
 					<groupId>org.apache.maven.plugins</groupId>
 					<artifactId>maven-deploy-plugin</artifactId>