Yadamu Code Restructuring Part 1.

Fixed #167 Fixed #168 Fixed #169 Fixed #170 Fixed #171 Fixed #172 Fixed #173 Fixed #174
markddrake · Feb 21, 2022 · 9e8d9d0 · 9e8d9d0
1 parent b99d730
commit 9e8d9d0
Show file tree

Hide file tree

Showing 211 changed files with 9,816 additions and 7,977 deletions.
diff --git a/docker/dockerfiles/linux/Dockerfile-GIT b/docker/dockerfiles/linux/Dockerfile-GIT
@@ -0,0 +1,36 @@
+FROM node:17.1.0
+WORKDIR /usr/src/YADAMU
+ADD https://download.oracle.com/otn_software/linux/instantclient/213000/oracle-instantclient-basic-21.3.0.0.0-1.x86_64.rpm .
+COPY src/package*.json ./
+RUN apt update \
+ && apt-get -y install libaio1 \
+ && apt-get -y install alien \
+ && alien -i --scripts ./oracle-instantclient-basic-21.3.0.0.0-1.x86_64.rpm \
+ && npm install --global electron \
+ && npm install
+COPY src src
+COPY bin bin
+ENV YADAMU_HOME=/usr/src/YADAMU
+ENV YADAMU_BIN=$YADAMU_HOME/bin
+ENV YADAMU_SRC=$YADAMU_HOME/src
+ENV PATH=$PATH:$YADAMU_BIN
+RUN cd bin \
+ && chmod +x $YADAMU_BIN/export.sh \
+ &&	ln -s $YADAMU_BIN/export.sh yadamuExport \
+ && chmod +x $YADAMU_BIN/import.sh \
+ &&	ln -s $YADAMU_BIN/import.sh yadamuImport \
+ && chmod +x $YADAMU_BIN/upload.sh \
+ &&	ln -s $YADAMU_BIN/upload.sh yadamuUpload \
+ && chmod +x $YADAMU_BIN/copy.sh \
+ &&	ln -s $YADAMU_BIN/copy.sh yadamuCopy \
+ && chmod +x $YADAMU_BIN/load.sh \
+ &&	ln -s $YADAMU_BIN/load.sh yadamuLoad \
+ && chmod +x $YADAMU_BIN/unload.sh \
+ &&	ln -s $YADAMU_BIN/unload.sh yadamuUnload \
+ && chmod +x $YADAMU_BIN/yadamu.sh \
+ &&	ln -s $YADAMU_BIN/yadamu.sh yadamu \
+ && chmod +x $YADAMU_BIN/decrypt.sh \
+ &&	ln -s $YADAMU_BIN/decrypt.sh yadamuDecrypt \
+ && chmod +x $YADAMU_BIN/encrypt.sh \
+ &&	ln -s $YADAMU_BIN/encrypt.sh yadamuEncrypt 
+ENTRYPOINT ["sleep","365d"]
diff --git a/docker/dockerfiles/linux/Dockerfile-GIT.dockerignore b/docker/dockerfiles/linux/Dockerfile-GIT.dockerignore
@@ -0,0 +1,3 @@
+*
+!bin
+!src
diff --git a/docker/dockerfiles/linux/GIT b/docker/dockerfiles/linux/GIT
diff --git a/docker/dockerfiles/linux/GIT.dockerignore b/docker/dockerfiles/linux/GIT.dockerignore
diff --git a/docker/dockerfiles/linux/docker-compose.yml b/docker/dockerfiles/linux/docker-compose.yml
@@ -263,14 +263,15 @@ services:
   MINIO-01: 
     container_name: MINIO-01
     image:  minio/minio 
-    command:  server /data
+    command:  server /data --console-address  ":9001"
     networks:
       - YADAMU-NET
     ports: 
       - "9000:9000"
+      - "9001:9001"
     environment:
-      MINIO_ACCESS_KEY: AKIAIOSFODNN7EXAMPLE
-      MINIO_SECRET_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+      MINIO_ROOT_USER: AKIAIOSFODNN7EXAMPLE
+      MINIO_ROOT_PASSWORD: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
     volumes:
       - MINIO-01-VOLUME:/data
     restart: always

diff --git a/docker/dockerfiles/linux/environment b/docker/dockerfiles/linux/environment
@@ -1,4 +1,4 @@
-FROM node:17.1.0
+FROM node:latest
 WORKDIR /usr/src/YADAMU
 ADD https://download.oracle.com/otn_software/linux/instantclient/213000/oracle-instantclient-basic-21.3.0.0.0-1.x86_64.rpm .
 COPY src/package*.json ./

diff --git a/qa/regression/dwShortRegression.json b/qa/regression/dwShortRegression.json
@@ -0,0 +1,50 @@
+{
+  "connections"                       : "connections.json"
+, "operation"                         : "dbRoundtrip"
+, "tasks"                             : "tasks.json"
+, "recreateSchema"                    : true
+, "createDirectory"                   : true
+, "verifyOperation"                   : true
+, "parameters": {
+    "REMOTE_STAGING_AREA"      : "/mnt/shared/stagingArea/db/vertica"
+    "REMOTE_STAGING_AREA"      : "/mnt/shared/stagingArea/db/vertica"
+  , "LOCAL_STAGING_AREA"       : "/usr/src/YADAMU/stagingArea/db/vertica"
+  , "INFINITY_MANAGEMENT"      : "NULLIFY"
+  , "CIRCLE_FORMAT"            : "CIRCLE"
+  }
+, "tests" : [{
+    "operation"          : "import"
+  , "source"             : "file-sRegress"
+  , "target"             : "vertica11"
+  , "parameters"         : {
+      "SOURCE_DIRECTORY" : "export/json/%location%/%mode%"
+    , "TARGET_DIRECTORY" : "json"
+    }
+  , "tasks": [         
+      "HR","BI"        
+	]                  
+  },{
+    "source": "vertica11"
+  , "target": "vertica11"
+  , "tasks": [
+       "HR","BI"
+    ]
+  },{
+    "operation"          : "import"
+  , "source"             : "file-sRegress"
+  , "target"             : "snow#1"
+  , "parameters"         : {
+      "SOURCE_DIRECTORY" : "export/json/%location%/%mode%"
+    , "TARGET_DIRECTORY" : "json"
+    }
+  , "tasks": [         
+      "HR","BI"        
+	]                  
+  },{
+    "source": "snow#1"
+  , "target": "snow#1"
+  , "tasks": [
+       "HR","BI"
+    ]
+  }]
+}
diff --git a/qa/regression/lostConnection.json b/qa/regression/lostConnection.json
@@ -16,10 +16,10 @@
    ,{"source":   "postgres#1", "target": "postgres#1",  "parameters" : { "PARALLEL" : 0, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "READER", "delay": 15000 }}
    ,{"source":   "mysql#1",    "target": "mysql#1",     "parameters" : { "PARALLEL" : 0, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "READER", "delay": 15000 }}
    ,{"source":   "mariadb#1",  "target": "mariadb#1",   "parameters" : { "PARALLEL" : 0, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "READER", "delay": 5000 }}  
-   ,{"source":   "oracle19c",  "target": "oracle19c",   "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "READER", "delay": 6000, "worker": 1}}
-   ,{"source":   "oracle18c",  "target": "oracle18c",   "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "READER", "delay": 6000, "worker": 1}}
-   ,{"source":   "oracle12c",  "target": "oracle12c",   "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "READER", "delay": 6000, "worker": 1}}
-   ,{"source":   "oracle11g",  "target": "oracle11g",   "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "READER", "delay": 6000, "worker": 1}}
+   ,{"source":   "oracle19c",  "target": "oracle19c",   "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "READER", "delay": 5000, "worker": 1}}
+   ,{"source":   "oracle18c",  "target": "oracle18c",   "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "READER", "delay": 5000, "worker": 1}}
+   ,{"source":   "oracle12c",  "target": "oracle12c",   "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "READER", "delay": 5000, "worker": 1}}
+   ,{"source":   "oracle11g",  "target": "oracle11g",   "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "READER", "delay": 5000, "worker": 1}}
    ,{"source":   "mssql19",    "target": "mssql19",     "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "READER", "delay": 3500, "worker": 1}}
    ,{"source":   "mssql17",    "target": "mssql17",     "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "READER", "delay": 3500, "worker": 1}}
    ,{"source":   "postgres#1", "target": "postgres#1",  "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "READER", "delay": 6000, "worker": 1}}
@@ -34,10 +34,10 @@
    ,{"source":   "postgres#1", "target": "postgres#1",  "parameters" : { "PARALLEL" : 0, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "WRITER", "delay": 15000 }}
    ,{"source":   "mysql#1",    "target": "mysql#1",     "parameters" : { "PARALLEL" : 0, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "WRITER", "delay": 15000 }}
    ,{"source":   "mariadb#1",  "target": "mariadb#1",   "parameters" : { "PARALLEL" : 0, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "WRITER", "delay": 10000 }}  
-   ,{"source":   "oracle19c",  "target": "oracle19c",   "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "WRITER", "delay": 6000, "worker": 0}}
-   ,{"source":   "oracle18c",  "target": "oracle18c",   "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "WRITER", "delay": 6000, "worker": 0}}
-   ,{"source":   "oracle12c",  "target": "oracle12c",   "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "WRITER", "delay": 6000, "worker": 0}}
-   ,{"source":   "oracle11g",  "target": "oracle11g",   "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "WRITER", "delay": 6000, "worker": 0}}
+   ,{"source":   "oracle19c",  "target": "oracle19c",   "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "WRITER", "delay": 5000, "worker": 0}}
+   ,{"source":   "oracle18c",  "target": "oracle18c",   "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "WRITER", "delay": 5000, "worker": 0}}
+   ,{"source":   "oracle12c",  "target": "oracle12c",   "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "WRITER", "delay": 5000, "worker": 0}}
+   ,{"source":   "oracle11g",  "target": "oracle11g",   "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "WRITER", "delay": 5000, "worker": 0}}
    ,{"source":   "mssql19",    "target": "mssql19",     "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "WRITER", "delay": 2500, "worker": 1}}
    ,{"source":   "mssql17",    "target": "mssql17",     "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "WRITER", "delay": 2500, "worker": 1}}
    ,{"source":   "postgres#1", "target": "postgres#1",  "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "ABORT" }, "tasks": ["WWI_Warehouse"], "kill": { "process": "WRITER", "delay": 1000, "worker": 1}}
@@ -88,10 +88,10 @@
    ,{"source":   "postgres#1", "target": "postgres#1",  "parameters" : { "PARALLEL" : 0, "ON_ERROR" : "SKIP"  }, "tasks": ["WWI_Warehouse"], "kill": { "process": "READER", "delay": 25000 }}
    ,{"source":   "mysql#1",    "target": "mysql#1",     "parameters" : { "PARALLEL" : 0, "ON_ERROR" : "SKIP"  }, "tasks": ["WWI_Warehouse"], "kill": { "process": "READER", "delay": 25000 }}
    ,{"source":   "mariadb#1",  "target": "mariadb#1",   "parameters" : { "PARALLEL" : 0, "ON_ERROR" : "SKIP"  }, "tasks": ["WWI_Warehouse"], "kill": { "process": "READER", "delay": 5000 }}  
-   ,{"source":   "oracle19c",  "target": "oracle19c",   "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "SKIP"  }, "tasks": ["WWI_Warehouse"], "kill": { "process": "WRITER", "delay": 6000, "worker": 0}}
-   ,{"source":   "oracle18c",  "target": "oracle18c",   "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "SKIP"  }, "tasks": ["WWI_Warehouse"], "kill": { "process": "WRITER", "delay": 6000, "worker": 0}}
-   ,{"source":   "oracle12c",  "target": "oracle12c",   "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "SKIP"  }, "tasks": ["WWI_Warehouse"], "kill": { "process": "WRITER", "delay": 6000, "worker": 0}}
-   ,{"source":   "oracle11g",  "target": "oracle11g",   "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "SKIP"  }, "tasks": ["WWI_Warehouse"], "kill": { "process": "WRITER", "delay": 6000, "worker": 0}}
+   ,{"source":   "oracle19c",  "target": "oracle19c",   "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "SKIP"  }, "tasks": ["WWI_Warehouse"], "kill": { "process": "WRITER", "delay": 5000, "worker": 0}}
+   ,{"source":   "oracle18c",  "target": "oracle18c",   "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "SKIP"  }, "tasks": ["WWI_Warehouse"], "kill": { "process": "WRITER", "delay": 5000, "worker": 0}}
+   ,{"source":   "oracle12c",  "target": "oracle12c",   "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "SKIP"  }, "tasks": ["WWI_Warehouse"], "kill": { "process": "WRITER", "delay": 5000, "worker": 0}}
+   ,{"source":   "oracle11g",  "target": "oracle11g",   "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "SKIP"  }, "tasks": ["WWI_Warehouse"], "kill": { "process": "WRITER", "delay": 5000, "worker": 0}}
    ,{"source":   "mssql19",    "target": "mssql19",     "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "SKIP"  }, "tasks": ["WWI_Warehouse"], "kill": { "process": "READER", "delay": 1500, "worker": 1}}
    ,{"source":   "mssql17",    "target": "mssql17",     "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "SKIP"  }, "tasks": ["WWI_Warehouse"], "kill": { "process": "READER", "delay": 1500, "worker": 1}}
    ,{"source":   "mysql#1",    "target": "mysql#1",     "parameters" : { "PARALLEL" : 2, "ON_ERROR" : "SKIP"  }, "tasks": ["WWI_Warehouse"], "kill": { "process": "READER", "delay": 1000, "worker": 1}}

diff --git a/qa/regression/vertica10DataTypes.json b/qa/regression/vertica10DataTypes.json
@@ -14,7 +14,7 @@
     "operation"       : "import"
   , "source"          : "file-staging"
   , "targets"         : [
-       "vertica10","vertica11","vertica11"
+       "vertica10","vertica11"
 	]
   , "verifyOperation" : true
   , "parameters"      : {

diff --git a/qa/regression/vertica11DataTypes.json b/qa/regression/vertica11DataTypes.json
@@ -78,7 +78,7 @@
       "postgresDataTypes"
 	]
   },{
-    "source"          : "vertica11"
+    "source"          : "vertica10"
   , "target"          : "vertica11"
   , "tasks": [ 
       "postgresDataTypes"

diff --git a/qa/regression/vertica11TestSuite.json b/qa/regression/vertica11TestSuite.json
@@ -83,7 +83,7 @@
       "oracle","mysql","mssql"
 	]
   },{
-    "source"            : "vertica11"
+    "source"            : "vertica10"
   , "target"            : "vertica11"
   , "tasks": [ 
       "oracle","mysql","mssql"
-Original file line number
+Diff line change
@@ -0,0 +1,3 @@
+    *
+    !bin
+    !src