From 7e18d803c25be3a64f5f2e284155b1ce3fbe6e44 Mon Sep 17 00:00:00 2001 From: hanpengs Date: Tue, 16 Jul 2024 10:27:57 +0800 Subject: [PATCH] whalealData --- docs/.DS_Store | Bin 8196 -> 6148 bytes docs/index.html | 2 +- docs/sitemap.xml.gz | Bin 127 -> 127 bytes .../InstallationRequirements/index.html | 5685 ++++++++++++++++ .../JDKInstallationDeployment/index.html | 5610 ++++++++++++++++ .../MYSQLInstallationDeployment/index.html | 5803 +++++++++++++++++ .../NginxInstallationDeployment/index.html | 5646 ++++++++++++++++ .../RedisInstallationDeployment/index.html | 5692 ++++++++++++++++ .../index.html | 5756 ++++++++++++++++ .../index.html | 5564 ++++++++++++++++ .../ColdTaskLogQuery/index.html | 5562 ++++++++++++++++ .../FileFullTextSearch/index.html | 5556 ++++++++++++++++ .../FileInspectionManagement/index.html | 5557 ++++++++++++++++ .../DataSourceManagement/index.html | 5598 ++++++++++++++++ .../DestinationSourceManagement/index.html | 5598 ++++++++++++++++ .../TableJobConfiguration/index.html | 5622 ++++++++++++++++ .../TaskConfiguration/index.html | 5601 ++++++++++++++++ .../HomepageDisplay/index.html | 5633 ++++++++++++++++ .../LoginPage/UserFirstLogin/index.html | 5503 ++++++++++++++++ .../LoginPage/UserRegistration/index.html | 5503 ++++++++++++++++ .../AbnormalJobExecutionStatistics/index.html | 5575 ++++++++++++++++ .../DataHistoricalFlow/index.html | 5535 ++++++++++++++++ .../StatisticalReports/JobDetails/index.html | 5575 ++++++++++++++++ .../RollbackRecordsStatistics/index.html | 5555 ++++++++++++++++ .../SystemAccessStatistics/index.html | 5596 ++++++++++++++++ .../TableJobExecutionStatistics/index.html | 5577 ++++++++++++++++ .../SystemManagement/ErrorLog/index.html | 5535 ++++++++++++++++ .../MenuManagement/index.html | 5597 ++++++++++++++++ .../SystemManagement/OperationLog/index.html | 5534 ++++++++++++++++ .../RoleManagement/index.html | 5619 ++++++++++++++++ .../SystemSettings/index.html | 5503 ++++++++++++++++ .../UserManagement/index.html | 5660 ++++++++++++++++ .../ColdTaskMonitoring/index.html | 5617 ++++++++++++++++ .../S3TaskMonitoring/index.html | 5617 ++++++++++++++++ .../TaskManagement/TaskScheduling/index.html | 5595 ++++++++++++++++ .../WarmTaskMonitoring/index.html | 5595 ++++++++++++++++ docs/whalealData/index.html | 5466 ++++++++++++++++ .../use cases/AbnormalTaskFeedback/index.html | 5506 ++++++++++++++++ .../AddColdDataFullLoadJob/index.html | 5510 ++++++++++++++++ .../AddColdDataIncrementalJob/index.html | 5512 ++++++++++++++++ .../use cases/AddDataSource/index.html | 5510 ++++++++++++++++ .../use cases/AddDestinationSource/index.html | 5510 ++++++++++++++++ .../use cases/AddS3FullLoadJob/index.html | 5511 ++++++++++++++++ .../use cases/AddS3IncrementalJob/index.html | 5514 ++++++++++++++++ .../AddWarmDataFullLoadJob/index.html | 5511 ++++++++++++++++ .../AddWarmDataIncrementalJob/index.html | 5512 ++++++++++++++++ .../ColdDataFullTextSearch/index.html | 5559 ++++++++++++++++ .../use cases/ColdDataWriteBack/index.html | 5511 ++++++++++++++++ .../use cases/ColdWorkDemo/index.html | 5778 ++++++++++++++++ .../use cases/CreateLoopTask/index.html | 5515 ++++++++++++++++ .../use cases/CreateManualTask/index.html | 5514 ++++++++++++++++ .../use cases/CreateSingleTask/index.html | 5515 ++++++++++++++++ .../ManuallyDeleteSourceData/index.html | 5513 ++++++++++++++++ .../use cases/RetryAbnormalTask/index.html | 5510 ++++++++++++++++ .../use cases/S3JobDemo/index.html | 5763 ++++++++++++++++ .../use cases/SMTPConfig/index.html | 5514 ++++++++++++++++ .../SystemDeleteSourceData/index.html | 5502 ++++++++++++++++ .../TaskExecutionMonitoring/index.html | 5502 ++++++++++++++++ .../use cases/UserLogin/index.html | 5501 ++++++++++++++++ .../use cases/UserRegistration/index.html | 5502 ++++++++++++++++ .../use cases/WarmJobDemo/index.html | 5763 ++++++++++++++++ 61 files changed, 323299 insertions(+), 1 deletion(-) create mode 100644 docs/whalealData/InstallationDeployment/InstallationRequirements/index.html create mode 100644 docs/whalealData/InstallationDeployment/JDKInstallationDeployment/index.html create mode 100644 docs/whalealData/InstallationDeployment/MYSQLInstallationDeployment/index.html create mode 100644 docs/whalealData/InstallationDeployment/NginxInstallationDeployment/index.html create mode 100644 docs/whalealData/InstallationDeployment/RedisInstallationDeployment/index.html create mode 100644 docs/whalealData/InstallationDeployment/Whaleal-dataInstallationDeployment/index.html create mode 100644 docs/whalealData/InstallationDeployment/ZookeeperInstallationDeployment/index.html create mode 100644 docs/whalealData/UserManual/ArchiveManagement/ColdTaskLogQuery/index.html create mode 100644 docs/whalealData/UserManual/ArchiveManagement/FileFullTextSearch/index.html create mode 100644 docs/whalealData/UserManual/ArchiveManagement/FileInspectionManagement/index.html create mode 100644 docs/whalealData/UserManual/ConfigurationManagement/DataSourceManagement/index.html create mode 100644 docs/whalealData/UserManual/ConfigurationManagement/DestinationSourceManagement/index.html create mode 100644 docs/whalealData/UserManual/ConfigurationManagement/TableJobConfiguration/index.html create mode 100644 docs/whalealData/UserManual/ConfigurationManagement/TaskConfiguration/index.html create mode 100644 docs/whalealData/UserManual/HomepageDisplay/HomepageDisplay/index.html create mode 100644 docs/whalealData/UserManual/LoginPage/UserFirstLogin/index.html create mode 100644 docs/whalealData/UserManual/LoginPage/UserRegistration/index.html create mode 100644 docs/whalealData/UserManual/StatisticalReports/AbnormalJobExecutionStatistics/index.html create mode 100644 docs/whalealData/UserManual/StatisticalReports/DataHistoricalFlow/index.html create mode 100644 docs/whalealData/UserManual/StatisticalReports/JobDetails/index.html create mode 100644 docs/whalealData/UserManual/StatisticalReports/RollbackRecordsStatistics/index.html create mode 100644 docs/whalealData/UserManual/StatisticalReports/SystemAccessStatistics/index.html create mode 100644 docs/whalealData/UserManual/StatisticalReports/TableJobExecutionStatistics/index.html create mode 100644 docs/whalealData/UserManual/SystemManagement/ErrorLog/index.html create mode 100644 docs/whalealData/UserManual/SystemManagement/MenuManagement/index.html create mode 100644 docs/whalealData/UserManual/SystemManagement/OperationLog/index.html create mode 100644 docs/whalealData/UserManual/SystemManagement/RoleManagement/index.html create mode 100644 docs/whalealData/UserManual/SystemManagement/SystemSettings/index.html create mode 100644 docs/whalealData/UserManual/SystemManagement/UserManagement/index.html create mode 100644 docs/whalealData/UserManual/TaskManagement/ColdTaskMonitoring/index.html create mode 100644 docs/whalealData/UserManual/TaskManagement/S3TaskMonitoring/index.html create mode 100644 docs/whalealData/UserManual/TaskManagement/TaskScheduling/index.html create mode 100644 docs/whalealData/UserManual/TaskManagement/WarmTaskMonitoring/index.html create mode 100644 docs/whalealData/index.html create mode 100644 docs/whalealData/use cases/AbnormalTaskFeedback/index.html create mode 100644 docs/whalealData/use cases/AddColdDataFullLoadJob/index.html create mode 100644 docs/whalealData/use cases/AddColdDataIncrementalJob/index.html create mode 100644 docs/whalealData/use cases/AddDataSource/index.html create mode 100644 docs/whalealData/use cases/AddDestinationSource/index.html create mode 100644 docs/whalealData/use cases/AddS3FullLoadJob/index.html create mode 100644 docs/whalealData/use cases/AddS3IncrementalJob/index.html create mode 100644 docs/whalealData/use cases/AddWarmDataFullLoadJob/index.html create mode 100644 docs/whalealData/use cases/AddWarmDataIncrementalJob/index.html create mode 100644 docs/whalealData/use cases/ColdDataFullTextSearch/index.html create mode 100644 docs/whalealData/use cases/ColdDataWriteBack/index.html create mode 100644 docs/whalealData/use cases/ColdWorkDemo/index.html create mode 100644 docs/whalealData/use cases/CreateLoopTask/index.html create mode 100644 docs/whalealData/use cases/CreateManualTask/index.html create mode 100644 docs/whalealData/use cases/CreateSingleTask/index.html create mode 100644 docs/whalealData/use cases/ManuallyDeleteSourceData/index.html create mode 100644 docs/whalealData/use cases/RetryAbnormalTask/index.html create mode 100644 docs/whalealData/use cases/S3JobDemo/index.html create mode 100644 docs/whalealData/use cases/SMTPConfig/index.html create mode 100644 docs/whalealData/use cases/SystemDeleteSourceData/index.html create mode 100644 docs/whalealData/use cases/TaskExecutionMonitoring/index.html create mode 100644 docs/whalealData/use cases/UserLogin/index.html create mode 100644 docs/whalealData/use cases/UserRegistration/index.html create mode 100644 docs/whalealData/use cases/WarmJobDemo/index.html diff --git a/docs/.DS_Store b/docs/.DS_Store index e2d81493f0e5ab163035eb6137e005c5de19331c..656795cb54a74feb88221990a9a63e3b9d465e0e 100644 GIT binary patch delta 107 zcmZp1XfcprU|?W$DortDU=RQ@Ie-{Mvv5r;6q~50$SANeU^g?Pz-AtST;|E^g)1i3 th%IL4;1Fa6DgpulZXn?bQophAJM(0I8BdUI1}2C}Aj26p$Mei#1_0S15o!Pc delta 130 zcmZoMXmOBWU|?W$DortDU;r^WfEYvza8E20o2aMAD7Z0TH}hr%jz7$c**Q2SHn1=X zZsuXhWoG7PD4*=aro_ki6R3RNe=uNRm|VtQy;+^-IWw08H&EOaWYT6qj_=Hq`9(Y@ S`}1&cFhVS0*c{I@hZz8XBOZGI diff --git a/docs/index.html b/docs/index.html index 14953dc4..9fbdfcc1 100644 --- a/docs/index.html +++ b/docs/index.html @@ -603,7 +603,7 @@

Whaleal

Whaleal Platform - + Whaleal Data Document Data Transfer diff --git a/docs/sitemap.xml.gz b/docs/sitemap.xml.gz index 450d58d93cfc0048b5f973a5008e176fcda44c50..786d5f69a6552c77845f86b6351746df46d0d952 100644 GIT binary patch delta 13 Ucmb=gXP58h;An84I+48s031RDEC2ui delta 13 Ucmb=gXP58h;9&SRc_MoS034+RSpWb4 diff --git a/docs/whalealData/InstallationDeployment/InstallationRequirements/index.html b/docs/whalealData/InstallationDeployment/InstallationRequirements/index.html new file mode 100644 index 00000000..3ccb9b2f --- /dev/null +++ b/docs/whalealData/InstallationDeployment/InstallationRequirements/index.html @@ -0,0 +1,5685 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + InstallationRequirements - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

InstallationRequirements

+ +

Installation Requirements

+

Hardware Requirements

+
    +
  • Operating System: Windows 10 or later, Linux distributions (such as Ubuntu, CentOS), MacOS.
    • +
  • Processor: Intel Core i5 or higher.
    • +
  • Memory: At least 8GB RAM.
    • +
  • Storage Space: At least 100GB of available disk space.
    • +
  • Network Adapter: Supports wired or wireless network connections.
    • +
+

Network Requirements

+

Network Access Requirements

+

Configure according to your specific needs.

+

Port Requirements

+

Specific ports need to be open (e.g., port 80 for HTTP communication, port used for program startup).

+

Software Requirements

+

Operating System Requirements

+
    +
  • Supports Windows Server 2016 or later.
    • +
  • Supports CentOS 7 or higher for Linux.
    • +
+

Browser Support

+
    +
  • Google Chrome version 80 or higher.
    • +
  • Mozilla Firefox version 75 or higher.
    • +
+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/InstallationDeployment/JDKInstallationDeployment/index.html b/docs/whalealData/InstallationDeployment/JDKInstallationDeployment/index.html new file mode 100644 index 00000000..23549991 --- /dev/null +++ b/docs/whalealData/InstallationDeployment/JDKInstallationDeployment/index.html @@ -0,0 +1,5610 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + JDKInstallationDeployment - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

JDKInstallationDeployment

+ +

JDK Installation and Deployment

+

It is recommended to install JDK 11.

+

1. Dependency Environment

+

For open-source JDK, font library support is required. If it is already present on the Linux system, there's no need to install it.

+
yum install fontconfig
+fc-cache --force
+fc-cache -f
+
+

2. Extract JDK Installation Package

+
tar -zxvf jdk-11.0.9_linux-x64_bin.tar.gz -C /usr/local/
+
+

3. Configure Environment Variables

+

Open the profile configuration file:

+
vi /etc/profile
+
+

Add the following configurations at the end of the file:

+
export JAVA_HOME=/usr/local/jdk-11.0.9
+export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
+export PATH=$PATH:$JAVA_HOME/bin
+
+

Refresh the configuration to make it effective:

+
source /etc/profile
+
+

4. Verification

+

Check the installed Java version:

+
java -version
+
+

Please note that when copying and pasting these commands, ensure that the formatting remains consistent, and adjust paths and filenames as needed for your system.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/InstallationDeployment/MYSQLInstallationDeployment/index.html b/docs/whalealData/InstallationDeployment/MYSQLInstallationDeployment/index.html new file mode 100644 index 00000000..1bddfec8 --- /dev/null +++ b/docs/whalealData/InstallationDeployment/MYSQLInstallationDeployment/index.html @@ -0,0 +1,5803 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + MYSQLInstallationDeployment - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

MYSQLInstallationDeployment

+ +

MySQL Installation and Deployment

+

It is recommended to use MySQL version 8.0.

+

Opening Specified Ports or Disabling Firewall

+
    +
  1. +

    View the ports that are already open: + bash + firewall-cmd --list-ports

    +
    2. +
  2. +

    Open a specified port (e.g., port 3306 for MySQL): + bash + firewall-cmd --zone=public --add-port=3306/tcp --permanent

    +
    3. +
  3. +

    Reload the firewall configuration: + bash + firewall-cmd --reload

    +
    4. +
  4. +

    Confirm the opened ports: + bash + firewall-cmd --list-ports

    +
    5. +
  5. +

    If needed, you can stop the firewall: + bash + systemctl stop firewalld

    +
    6. +
  6. +

    Check the firewall status: + bash + systemctl status firewalld

    +
    7. +
+

Basic Environment Preparation

+
    +
  1. +

    Create a user and group for MySQL: + bash + groupadd mysql + useradd -r -g mysql -s /sbin/nologin mysql

    +
    2. +
  2. +

    Install dependencies for MySQL: + bash + yum install -y libncurses* libaio* lrzsz*

    +
    3. +
  3. +

    Extract the MySQL installation package: + bash + tar -xvf mysql-8.0.28-linux-glibc2.12-x86_64.tar -C /usr/local/

    +
    4. +
  4. +

    Rename the extracted directory: + bash + mv mysql-8.0.28-linux-glibc2.12-x86_64/ mysql

    +
    5. +
  5. +

    Create required directories: + bash + cd /usr/local/mysql/ + mkdir data

    +
    6. +
  6. +

    Change directory ownership: + bash + chown -R mysql:mysql /usr/local/mysql/

    +
    7. +
+

Deploy MySQL Service

+
    +
  1. +

    Initialize the database: + bash + /usr/local/mysql/bin/mysqld --user=mysql --basedir=/usr/local/mysql/ --datadir=/usr/local/mysql/data/ --initialize

    +
    2. +
  2. +

    Edit my.cnf configuration: + Create/Edit the configuration file /etc/my.cnf and add the following content: + ``` + [mysqld] + basedir=/usr/local/mysql + datadir=/usr/local/mysql/data + socket=/usr/local/mysql/data/mysql.sock + bind-address = 0.0.0.0 + user=root + port=3306 + log-bin=mysql-bin + server-id=1 + max_connections=2048 + character-set-server=utf8 + default-storage-engine=INNODB

    +
    3. +
+

[client] + socket=/usr/local/mysql/data/mysql.sock + ```

+
    +
  1. +

    Configure environment variables: + bash + echo "export PATH=$PATH:/usr/local/mysql/bin" >> /etc/profile + source /etc/profile

    +
    2. +
  2. +

    Configure startup script: + ```bash + cp /usr/local/mysql/support-files/mysql.server /etc/rc.d/init.d/mysqld + chmod +x /etc/rc.d/init.d/mysqld

    +
    3. +
+

cat > /lib/systemd/system/mysqld.service <<EOF + [Unit] + Description=mysqld + After=network.target

+

[Service] + Type=forking + ExecStart=/etc/rc.d/init.d/mysqld start + ExecReload=/etc/rc.d/init.d/mysqld restart + ExecStop=/etc/rc.d/init.d/mysqld stop + PrivateTmp=true

+

[Install] + WantedBy=multi-user.target + EOF + ```

+
    +
  1. +

    Reload systemd configuration: + bash + systemctl daemon-reload

    +
    2. +
  2. +

    Set MySQL to start on boot: + bash + systemctl enable mysqld

    +
    3. +
  3. +

    Start MySQL: + bash + systemctl start mysqld

    +
    4. +
  4. +

    Check if MySQL port is active: + bash + netstat -tunlp | grep 3306

    +
    5. +
+

Configure Password for Remote Connection

+
    +
  1. +

    Enter the printed password to log in to MySQL: + bash + mysql -u root -p

    +
    2. +
  2. +

    After logging in, change the root password: + sql + ALTER USER 'root'@'localhost' IDENTIFIED BY '123456';

    +
    3. +
  3. +

    Check user information: + sql + select user, host, ssl_type from mysql.user; + use mysql;

    +
    4. +
  4. +

    Update the host field to % to allow remote connections: + sql + update user set host = '%' where user = 'root';

    +
    5. +
  5. +

    Refresh privileges: + sql + flush privileges;

    +
    6. +
+

Adding Archive Platform Fields

+
    +
  1. +

    Log in to MySQL: + bash + mysql -u root -p

    +
    2. +
  2. +

    Create a database: + sql + create database filing;

    +
    3. +
  3. +

    Add data from the provided SQL file: + sql + use filing; + source /usr/local/filing.sql;

    +
    4. +
  4. +

    Check the added tables: + sql + use filing; + show tables;

    +
    5. +
+

As always, ensure that you adapt paths, filenames, and other specifics to match your system's configuration.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/InstallationDeployment/NginxInstallationDeployment/index.html b/docs/whalealData/InstallationDeployment/NginxInstallationDeployment/index.html new file mode 100644 index 00000000..e4b76553 --- /dev/null +++ b/docs/whalealData/InstallationDeployment/NginxInstallationDeployment/index.html @@ -0,0 +1,5646 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + NginxInstallationDeployment - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

NginxInstallationDeployment

+ +

Nginx Installation and Deployment

+

Opening Specified Ports or Disabling Firewall

+
    +
  1. +

    View the ports that are already open: + bash + firewall-cmd --list-ports

    +
    2. +
  2. +

    Open a specified port (e.g., port 80 for Nginx): + bash + firewall-cmd --zone=public --add-port=80/tcp --permanent

    +
    3. +
  3. +

    Reload the firewall configuration: + bash + firewall-cmd --reload

    +
    4. +
  4. +

    Confirm the opened ports: + bash + firewall-cmd --list-ports

    +
    5. +
  5. +

    If needed, you can stop the firewall: + bash + systemctl stop firewalld

    +
    6. +
  6. +

    Check the firewall status: + bash + systemctl status firewalld

    +
    7. +
+

Installation and Deployment

+
    +
  1. +

    Extract the Nginx installation package: + bash + tar -zxvf nginx-1.16.1.tar.gz -C /usr/local/

    +
    2. +
  2. +

    Install dependencies: + bash + yum install -y pcre pcre-devel + yum install -y zlib zlib-devel

    +
    3. +
  3. +

    Configure the installation path: + bash + cd /usr/local/nginx-1.16.1 + ./configure --prefix=/usr/local/nginx

    +
    4. +
  4. +

    Compile Nginx: + bash + make && make install

    +
    5. +
  5. +

    Configure local hostname resolution: + Edit /etc/hosts and add an entry for your local domain: + ip cloud.whalealmg.com

    +
    6. +
  6. +

    Edit the Nginx configuration file: + ```nginx + server { + listen 80; + server_name cloud.whalealmg.com;

    +

    location / { + root /usr/local/nginx/html/dist/; + index index.html index.htm; + try_files $uri $uri/ /index.html; + }

    +

    location /filingAdmin/ { + proxy_pass http://127.0.0.1:8000/; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-Port $server_port; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + }

    +

    location ~ .*.(js|css|jpg|jpeg|gif|png|ico|pdf|txt)$ { + root /usr/local/nginx/html/dist/; + index index.html index.htm; + }

    +

    error_page 500 502 503 504 /50x.html; + location = /50x.html { + root html; + } + } + ```

    +
    7. +
  7. +

    Start the Nginx service: + bash + /usr/local/nginx/sbin/nginx

    +
    8. +
+

Make sure to adjust paths, domain names, and other configurations as needed for your specific environment.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/InstallationDeployment/RedisInstallationDeployment/index.html b/docs/whalealData/InstallationDeployment/RedisInstallationDeployment/index.html new file mode 100644 index 00000000..e3c5ce6d --- /dev/null +++ b/docs/whalealData/InstallationDeployment/RedisInstallationDeployment/index.html @@ -0,0 +1,5692 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + RedisInstallationDeployment - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

RedisInstallationDeployment

+ +

Redis Installation and Deployment

+

Opening Specified Ports or Disabling Firewall

+
    +
  1. +

    View the ports that are already open: + bash + firewall-cmd --list-ports

    +
    2. +
  2. +

    Open a specified port (e.g., port 6379 for Redis): + bash + firewall-cmd --zone=public --add-port=6379/tcp --permanent

    +
    3. +
  3. +

    Reload the firewall configuration: + bash + firewall-cmd --reload

    +
    4. +
  4. +

    Confirm the opened ports: + bash + firewall-cmd --list-ports

    +
    5. +
  5. +

    If needed, you can stop the firewall: + bash + systemctl stop firewalld

    +
    6. +
  6. +

    Check the firewall status: + bash + systemctl status firewalld

    +
    7. +
+

Installation and Deployment

+
    +
  1. +

    Extract the Redis installation package: + bash + tar -zxvf redis-4.0.9.tar.gz -C /usr/local/

    +
    2. +
  2. +

    Rename the extracted folder: + bash + mv redis-4.0.9 redis

    +
    3. +
  3. +

    Install the required dependencies (e.g., GCC): + bash + yum install gcc -y

    +
    4. +
  4. +

    Compile the Redis files: + bash + cd /usr/local/redis + make && make install

    +
    5. +
+

Edit Configuration File

+
    +
  1. +

    Edit the Redis configuration file: + bash + vi redis.conf

    +
    2. +
  2. +

    Set a password (e.g., "123456"): + ```conf + # Before + # requirepass foobared

    +
    3. +
+

# After + requirepass 123456 + ```

+
    +
  1. Enable background daemon mode: + ```conf + # Before + # daemonize no
    2. +
+

# After + daemonize yes + ```

+
    +
  1. Allow remote access: + ```conf + # Before + # bind 127.0.0.1
    2. +
+

# After + bind 0.0.0.0 + ```

+
    +
  1. Save the configuration file and exit the editor.
    2. +
+

Start Redis

+
    +
  1. +

    Start Redis using the modified configuration file: + bash + redis-server /usr/local/redis/redis.conf

    +
    2. +
  2. +

    Validate the Redis server is running: + bash + redis-cli

    +
    3. +
+

Make sure to adjust paths, passwords, and other configurations as needed for your specific environment.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/InstallationDeployment/Whaleal-dataInstallationDeployment/index.html b/docs/whalealData/InstallationDeployment/Whaleal-dataInstallationDeployment/index.html new file mode 100644 index 00000000..668e93a7 --- /dev/null +++ b/docs/whalealData/InstallationDeployment/Whaleal-dataInstallationDeployment/index.html @@ -0,0 +1,5756 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + Whaleal-dataInstallationDeployment - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

Whaleal-dataInstallationDeployment

+ +

Installation and Deployment of Whaleal-data

+

High Availability Deployment

+

To achieve high availability, deploy the service on multiple machines and distribute traffic through a load balancer to balance and share the requests. Common load balancing algorithms include round-robin, least connections, and hash algorithms. Use multiple servers with the same configuration to maintain system continuity by having other servers take over in case of a failure. Common redundancy backup modes include master-slave mode, active-active mode, and N+1 mode.

+

Package Deployment

+

Frontend Service Startup

+
    +
  1. After compiling the source code, generate the "dist" distribution package.
    2. +
  2. Send the "dist" package to the server.
    3. +
  3. Path: The installation path configured in the Nginx configuration.
    4. +
+

Restart Nginx

+
/usr/local/nginx/sbin/nginx -s reload -t
+
+

Backend Service Startup

+
    +
  1. After compiling the source code, generate the "filing-system-0.0.1-SNAPSHOT.jar" distribution package.
    2. +
  2. Upload the distribution package to the server.
    3. +
  3. Edit the configuration file "application.yml".
    4. +
+

Configuration File Content

+
# Application server port
+server:
+  port: 8000
+
+# Database and other configurations...
+
+

Start the Service

+
nohup java -jar -Xms2048M -Xmx20000M -XX:PermSize=768M -XX:MaxPermSize=1536M -server -jar filing-system-0.0.1-SNAPSHOT.jar --spring.config.location=application.yml --jasypt.encryptor.password=SfXlqZmK4P257 &
+
+

Check Logs for Successful Startup

+
tail -f nohup.out
+
+

Docker Container Deployment

+
    +
  1. Navigate to the directory containing the docker-compose.yml file.
    2. +
  2. Start the service using the command: docker-compose up -d.
    3. +
+

After the Docker service starts successfully, you can view the logs using the command: docker logs -f root_whaleal-data_1.

+

For local access, bind the server's IP with the domain name in the hosts file using: sudo sh -c 'echo "docker_server_ip whaleal-data.com" >> /etc/hosts'.

+

Access the Whaleal-data service:

+
    +
  • Web URL: http://docker_server_ip or http://whaleal-data.com
    • +
  • Initial login:
      +
    • User: "admin"
      • +
    • Password: "123456"
      • +
    • The system will force you to change the password upon first login.
      • +
    +
    • +
+

Tips: +Cold Data Archiving: +The default path for cold data archiving is /whalealdb. For Docker, the service is mapped to an external path /opt/whalealdb.

+

Quick Access

+

Start the Whaleal-data service using Docker containers. This service depends on mysql, mongodb, redis, and zookeeper services. It runs in a local browser through the nginx service proxy.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/InstallationDeployment/ZookeeperInstallationDeployment/index.html b/docs/whalealData/InstallationDeployment/ZookeeperInstallationDeployment/index.html new file mode 100644 index 00000000..41b27f67 --- /dev/null +++ b/docs/whalealData/InstallationDeployment/ZookeeperInstallationDeployment/index.html @@ -0,0 +1,5564 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + ZookeeperInstallationDeployment - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

ZookeeperInstallationDeployment

+ +

Zookeeper Installation and Deployment

+

Opening Specific Ports or Disabling Firewall

+
    +
  1. Check already opened ports: firewall-cmd --list-ports
    2. +
  2. Open a specific port: firewall-cmd --zone=public --add-port=2181/tcp --permanent
    3. +
  3. Reload firewall configuration: firewall-cmd --reload
    4. +
  4. Confirm opened ports: firewall-cmd --list-ports
    5. +
  5. Stop the firewall: systemctl stop firewalld
    6. +
  6. Check firewall status: systemctl status firewalld
    7. +
+

Installation and Deployment

+
    +
  1. Unpack the installation package: tar -zxvf apache-zookeeper-3.6.1-bin.tar.gz -C /usr/local/
    2. +
  2. Rename the extracted folder: mv apache-zookeeper-3.6.1-bin/ zookeeper
    3. +
  3. Start Zookeeper: /usr/local/zookeeper/bin/zkServer.sh start /usr/local/zookeeper/conf/zoo_sample.cfg
    4. +
  4. Verify Zookeeper status: /usr/local/zookeeper/bin/zkServer.sh status /usr/local/zookeeper/conf/zoo_sample.cfg
    5. +
+

This installation guide provides steps for deploying Zookeeper, opening the required ports, and starting the service. Make sure to follow each step carefully to ensure a successful deployment.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/UserManual/ArchiveManagement/ColdTaskLogQuery/index.html b/docs/whalealData/UserManual/ArchiveManagement/ColdTaskLogQuery/index.html new file mode 100644 index 00000000..2bb697f2 --- /dev/null +++ b/docs/whalealData/UserManual/ArchiveManagement/ColdTaskLogQuery/index.html @@ -0,0 +1,5562 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + ColdTaskLogQuery - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

ColdTaskLogQuery

+ +

Log Query (Cold)

+

Click on "Log Query (Cold)" under the "Archive Management" menu to query all file archiving log information. This page includes a search button, search condition fields, and a "Rewrite" button for each task.

+

image-20230620165056555

+
Searching
+

The green button at the top is the search button. The left-hand side condition fields are used for specifying search criteria. After filling in the conditions, click the search button to filter the desired tasks.

+
File Rewriting
+

Click on the yellow "File Rewrite" button behind a task to initiate a file rewriting operation. This opens a page where you can click "Create Rewrite Task" to create the rewrite task.

+

image-20230620165406520

+

Creating a Rewrite Task

+

Clicking the "Create Rewrite Task" button opens the following form. Fill in the required information and click "Confirm."

+

image-20230620165554471

+

Delete Temporary Table

+

After creating the rewrite task, you can start the rewriting process. Once it's completed, you can click the "Delete Temporary Table" button. A confirmation prompt will appear. If you want to proceed with deletion, click "Execute." If not, click "Cancel."

+

image-20230620165618947

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/UserManual/ArchiveManagement/FileFullTextSearch/index.html b/docs/whalealData/UserManual/ArchiveManagement/FileFullTextSearch/index.html new file mode 100644 index 00000000..00efbe3a --- /dev/null +++ b/docs/whalealData/UserManual/ArchiveManagement/FileFullTextSearch/index.html @@ -0,0 +1,5556 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + FileFullTextSearch - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

FileFullTextSearch

+ +

File Full-Text Search

+

Click on "File Full-Text Search" under the "Archive Management" menu to query all file full-text search log information. Click the "Create Search Task" button to create a file full-text search task. The system will perform a comprehensive search for files based on the search criteria, and for the found records, you can perform file rewriting operations.

+

image-20230620170814233

+
Searching
+

The green button at the top is the search button. The left-hand side condition fields are optional. After filling in the conditions, click the search button to filter the desired search content.

+
Creating a Search Task
+

Click the blue "Create Search Task" button to open the following form. Fill in the required information and click "Confirm."

+

image-20230620171123012

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/UserManual/ArchiveManagement/FileInspectionManagement/index.html b/docs/whalealData/UserManual/ArchiveManagement/FileInspectionManagement/index.html new file mode 100644 index 00000000..4c121360 --- /dev/null +++ b/docs/whalealData/UserManual/ArchiveManagement/FileInspectionManagement/index.html @@ -0,0 +1,5557 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + FileInspectionManagement - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

FileInspectionManagement

+ +

File Inspection Management

+

Click on "File Inspection Management" under the "Archive Management" menu to query all file inspection log information. Click the "Create Inspection Task" button to create a file inspection task. The system will perform the necessary file checking and verification logic. For files with exceptions, you can manually update the file path or perform a re-archive operation.

+

image-20230620170305646

+
Searching
+

The green button at the top is the search button. The left-hand side has condition fields that you can optionally fill in. After filling in the conditions, click the search button to filter the desired inspection.

+
Creating an Inspection Task
+

Click the blue "Create Inspection Task" button to open the following form. Fill in the required information and click "Confirm" to generate inspection data for the corresponding target source of warm data. After completing the task, click "View Details" to see the inspection content as shown in the second image.

+

image-20230620170622316

+

image-20230620170741632

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/UserManual/ConfigurationManagement/DataSourceManagement/index.html b/docs/whalealData/UserManual/ConfigurationManagement/DataSourceManagement/index.html new file mode 100644 index 00000000..82a90efe --- /dev/null +++ b/docs/whalealData/UserManual/ConfigurationManagement/DataSourceManagement/index.html @@ -0,0 +1,5598 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + DataSourceManagement - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

DataSourceManagement

+ +

Data Source Management

+

Adding a Data Source

+

To add a new data source, navigate to "Data Source Management" under the "Configuration Management" menu. Here, you can view all the information about existing data sources. Click on the "New" button in the first image to bring up the form shown in the second image, where you can input the details of the new data source. Data sources can include MongoDB, MYSQL, Oracle, and DB2. Provide the required username and password for connecting to the database. The "Options" field contains optional settings for configuring specific connection options for the database. Click the "Test" button to verify if the provided user information can connect to the database. Finally, click "Confirm" to save the data source.

+

image-20230619170823978

+

image-20230619171346159

+

Modifying a Data Source

+

Click the blue button in the right-hand side action column to edit a data source. The form for editing a data source is similar to the form for adding a data source. After making the necessary modifications, click "Test." If the test is successful, click "Confirm" to save the changes.

+

image-20230619171443926

+

Deleting a Data Source

+

Click the red button in the right-hand side action column to delete a data source. A confirmation prompt will appear. If you intend to delete the data source, click "Confirm." If you clicked by mistake, you can click "Cancel."

+

image-20230619171652736

+

Binding Data Source to Users

+

Navigate to "User Management" under "System Management" to display all user information. Click on the user you want to operate on, and then check the data sources you want to bind to this user on the right side. Afterward, click "Save" to bind the selected data sources to the user.

+

image-20230619171903840

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/UserManual/ConfigurationManagement/DestinationSourceManagement/index.html b/docs/whalealData/UserManual/ConfigurationManagement/DestinationSourceManagement/index.html new file mode 100644 index 00000000..83244635 --- /dev/null +++ b/docs/whalealData/UserManual/ConfigurationManagement/DestinationSourceManagement/index.html @@ -0,0 +1,5598 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + DestinationSourceManagement - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

DestinationSourceManagement

+ +

Target Source Management

+

Adding Warm/Cold/S3 Data Target Sources

+

To add a new target source, navigate to "Target Source Management" under the "Configuration Management" menu. If you want to add a warm data target source, click on "Warm Data Target Source." For cold data target sources, click on "Cold Data File Source." Similarly, for S3 target sources, click on "S3." Then, click the "New" button to bring up the form for adding a new target source. The process for cold data and S3 is similar to adding a data source. After completing the necessary details, click "Confirm" to save the target source. For cold data file sources, click "Confirm" after filling in the required information.

+

image-20230619172342937

+

image-20230619172709222

+

Modifying Warm/Cold/S3 Data Target Sources

+

Click the blue button to edit a target source. The process for modifying warm data target sources and S3 is similar to modifying a data source. For cold data file sources, modify the path and click "Confirm" to save the changes.

+

image-20230619173000216

+

Deleting Warm/Cold/S3 Data Target Sources

+

Click the red button to delete a target source. A confirmation prompt will appear. If you intend to delete the target source, click "Confirm." If not, click "Cancel."

+

image-20230619173144766

+

Binding Warm/Cold/S3 Data Target Sources to Users

+

The process of binding target sources is similar to binding data sources. If you want to bind a target source, click on the target source, and then check the relevant sources. The process is the same for file sources and S3. After that, click "Save" to grant the user access to the selected sources.

+

image-20230619173446875

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/UserManual/ConfigurationManagement/TableJobConfiguration/index.html b/docs/whalealData/UserManual/ConfigurationManagement/TableJobConfiguration/index.html new file mode 100644 index 00000000..01f0225e --- /dev/null +++ b/docs/whalealData/UserManual/ConfigurationManagement/TableJobConfiguration/index.html @@ -0,0 +1,5622 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + TableJobConfiguration - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

TableJobConfiguration

+ +

Table Job Configuration

+

Adding Warm/Cold/S3 Jobs

+

To configure a new table job, navigate to "Table Job Configuration" under the "Configuration Management" menu. If you want to configure a warm data table job, click on "Warm Data Table Job Configuration." For cold data table job configuration, click on "Cold Data Table Job Configuration." Similarly, for S3 table job configuration, click on "S3 Table Job Configuration." Click the type that you want to add, and then a table will appear displaying the respective job information. The process for adding each type of job is similar.

+

After selecting the data source and target source, you can choose options such as consistency verification, archiving method, and data processing mode. If you choose consistency verification, you can fill in the verification percentage. When the archiving mode is "Full Update," you don't need to enter SQL conditions, as the platform archives the entire table data directly. When the archiving mode is "Incremental Update," an input field for SQL conditions will appear along with a "Custom SQL Configuration" button, as shown in the third image. If you want to configure custom SQL, you can click the button to enter the visual configuration interface for table fields, as shown in the fourth image. When both the data source and target source are MongoDB, you can choose Gridfs. If Gridfs is enabled, MongoDB's Gridfs data type will be synchronized.

+

image-20230619173749218

+

image-20230619175831400

+

image-20230619180055920

+

image-20230619180204209

+

Editing Warm/Cold/S3 Jobs

+

Click the blue button on the right to edit a job. After making the necessary changes to the job, click "Save" to save the modifications.

+

image-20230619180801794

+

Deleting Warm/Cold/S3 Jobs

+

Click the red button on the right to delete a job. A confirmation prompt will appear. If you intend to delete the job, click "Confirm." If not, click "Cancel."

+

image-20230619181955829

+

Searching Warm/Cold/S3 Jobs

+

Click the search button at the top and fill in the first three condition boxes to filter all table job configurations that meet the specified criteria on the platform.

+

image-20230619182228673

+

Viewing Warm/Cold/S3 Jobs

+

Click the "View" button on the right to see the job configuration details. However, you won't be able to perform any actions in this view.

+

image-20230619182140322

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/UserManual/ConfigurationManagement/TaskConfiguration/index.html b/docs/whalealData/UserManual/ConfigurationManagement/TaskConfiguration/index.html new file mode 100644 index 00000000..4bda5223 --- /dev/null +++ b/docs/whalealData/UserManual/ConfigurationManagement/TaskConfiguration/index.html @@ -0,0 +1,5601 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + TaskConfiguration - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

TaskConfiguration

+ +

Task Configuration

+

Adding Warm/Cold/S3 Tasks

+

To configure a new task, go to "Task Configuration" under the "Configuration Management" menu. Click on "Warm Data Task Configuration" to view all warm data archiving task information. Click on "Cold Data Task Configuration" to view all cold data archiving task information. Similarly, click on "S3 Task Configuration" to view all S3 archiving task information. After selecting the type of task you want to add, click the "Add" button to bring up the table shown in the second image. Click "Add Job" to show the already configured table job configuration, and then associate the desired table jobs with the task configuration, as shown in the third image. A task can have multiple table jobs associated with it. You can choose between manual tasks, recurring tasks, and one-time tasks. For one-time tasks, you can set the execution time using a Cron expression, while recurring tasks must have a Cron expression for scheduling. Since a task can have multiple table jobs, you can configure the execution mode to be either serial or parallel. You can also set the task timeout and configure the number of retries in case of failure. Additionally, you can set up email notifications for task success or failure. Once the configuration is complete, as shown in the fourth image, click "Confirm." After configuration, the task needs to be reviewed by a management user.

+

image-20230619182516563

+

image-20230619182801452

+

image-20230619183021765

+

image-20230619183158936

+

image-20230619183350447

+

Editing Warm/Cold/S3 Tasks

+

Click the edit button to open the table shown in the first image. This task configuration can be edited or deleted until it is reviewed by a management user. Once reviewed, the task cannot be edited or deleted. After editing the task, click "Confirm" to save the changes.

+

image-20230619183538325

+

Deleting Warm/Cold/S3 Tasks

+

Before being reviewed by a management user, tasks can be edited or deleted. Click the red "Delete" button to show a confirmation prompt. If you are sure you want to delete the task, click "Delete." If not, click "Cancel."

+

image-20230619183730879

+

Searching Warm/Cold/S3 Tasks

+

There are two condition boxes before the search button. Fill in the criteria and click "Search" to filter the task configurations that match the specified criteria.

+

image-20230619183839994

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/UserManual/HomepageDisplay/HomepageDisplay/index.html b/docs/whalealData/UserManual/HomepageDisplay/HomepageDisplay/index.html new file mode 100644 index 00000000..e6979da2 --- /dev/null +++ b/docs/whalealData/UserManual/HomepageDisplay/HomepageDisplay/index.html @@ -0,0 +1,5633 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + HomepageDisplay - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

HomepageDisplay

+ +

Homepage Display

+

The homepage displays various statistics and information related to the system's activities. Users can customize the time range for which they want to see the data. Here's a breakdown of the different sections on the homepage:

+

image-20230619163709752

+

Task Execution Statistics

+

The "Task Execution Statistics" pie chart depicts the distribution of task execution results within a specific time range. Task execution statuses include running (in progress), succeeded (successfully completed), failed (execution failed), and other (remaining types).

+

Archiving Capacity Statistics

+

The "Archiving Capacity Statistics" section displays the total size of data synchronized from different source types to various destination types. The first column represents source types, the second column represents destination types, and the third column represents either cold data destination files or S3. The capacity statistics depend on the source and destination types, and certain combinations are supported based on compatibility.

+

User Activity Statistics

+

The "User Activity Statistics" section scrolls through and displays the actions taken by a user during a specific time period. It showcases login actions, task executions, and other relevant activities performed by users.

+

Table Job Statistics

+

The "Table Job Statistics" section presents a line chart depicting the quantity of different types of table jobs executed within the chosen time range. This helps users observe recent trends in task execution. Users can select different job types from the dropdown menu to view the corresponding line chart.

+

Business Access (Number of Sources)

+

The "Business Access (Number of Sources)" section displays the number of data source accesses made to the platform during the specified time period. It shows the growth trend of data source accesses over time. Similarly, users can choose different data source types from the dropdown menu.

+

Total Archiving Volume

+

The "Total Archiving Volume" section presents a line chart illustrating the total data volume archived by the platform within the selected time range. This helps users track the growth trend of archiving volume over time.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/UserManual/LoginPage/UserFirstLogin/index.html b/docs/whalealData/UserManual/LoginPage/UserFirstLogin/index.html new file mode 100644 index 00000000..925afa25 --- /dev/null +++ b/docs/whalealData/UserManual/LoginPage/UserFirstLogin/index.html @@ -0,0 +1,5503 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + UserFirstLogin - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

UserFirstLogin

+ +

First-Time User Login

+

During the system initialization, the administrator account is set as "admin" with the password "123456". When logging in for the first time, users will be prompted to change their password. After modifying the password during the initial login, users can proceed to log in with the new password.

+

image-20230619161736540

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/UserManual/LoginPage/UserRegistration/index.html b/docs/whalealData/UserManual/LoginPage/UserRegistration/index.html new file mode 100644 index 00000000..51767f56 --- /dev/null +++ b/docs/whalealData/UserManual/LoginPage/UserRegistration/index.html @@ -0,0 +1,5503 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + UserRegistration - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

UserRegistration

+ +

User Registration

+

The platform supports self-registration of user accounts. After registering, the administrator user can assign resource permissions to new users.

+

image-20230619154745394

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/UserManual/StatisticalReports/AbnormalJobExecutionStatistics/index.html b/docs/whalealData/UserManual/StatisticalReports/AbnormalJobExecutionStatistics/index.html new file mode 100644 index 00000000..9392a87b --- /dev/null +++ b/docs/whalealData/UserManual/StatisticalReports/AbnormalJobExecutionStatistics/index.html @@ -0,0 +1,5575 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + AbnormalJobExecutionStatistics - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

AbnormalJobExecutionStatistics

+ +

Abnormal Job Execution Statistics

+

Clicking on the "Abnormal Job Execution Statistics" under the "Statistics Report" menu allows you to query the archival information of all abnormal table jobs. This page includes a search button, search criteria fields, a clear criteria button, and an export button.

+

image-20230621111904900

+ +

The green button in the image above is the search button. On the left side, there's a criteria field. After filling in the filtering criteria, click "Search" to filter out abnormal table job statistics that match the criteria.

+
Clear
+

The black button is the clear button, which clears the time criteria fields.

+
Export
+

The yellow button is the export button, which allows you to export the abnormal table job statistics information as an XLSX file.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/UserManual/StatisticalReports/DataHistoricalFlow/index.html b/docs/whalealData/UserManual/StatisticalReports/DataHistoricalFlow/index.html new file mode 100644 index 00000000..7a7f0906 --- /dev/null +++ b/docs/whalealData/UserManual/StatisticalReports/DataHistoricalFlow/index.html @@ -0,0 +1,5535 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + DataHistoricalFlow - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

DataHistoricalFlow

+ +

Data Historical Flow

+

Clicking on the "Data Historical Flow" under the "Statistics Report" menu displays the data source flow within the platform. This page includes a search button, search criteria fields, and a clear criteria button.

+

image-20230621113944243

+ +

The green button in the image above is the search button. On the left side, there are three criteria fields. After filling in the filtering criteria, click "Search" to filter out data flows that match the criteria.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/UserManual/StatisticalReports/JobDetails/index.html b/docs/whalealData/UserManual/StatisticalReports/JobDetails/index.html new file mode 100644 index 00000000..c7c1de89 --- /dev/null +++ b/docs/whalealData/UserManual/StatisticalReports/JobDetails/index.html @@ -0,0 +1,5575 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + JobDetails - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

JobDetails

+ +

Job Details

+

Clicking on "Job Details" under the "Statistics Report" menu displays the detailed information about jobs within the platform. This page includes a search button, search criteria fields, and an export button.

+

image-20230621113251137

+ +

The green button in the image above is the search button. On the left side, there are three criteria fields. After filling in the filtering criteria, click "Search" to filter out job details that match the criteria.

+
Export
+

The yellow button is the export button, which allows you to export the job details statistics information as an xlsx file.

+ +

For each job detail, there is a "Link" button. Clicking the link button will navigate you to the task monitoring page for that specific job.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/UserManual/StatisticalReports/RollbackRecordsStatistics/index.html b/docs/whalealData/UserManual/StatisticalReports/RollbackRecordsStatistics/index.html new file mode 100644 index 00000000..d927ad9b --- /dev/null +++ b/docs/whalealData/UserManual/StatisticalReports/RollbackRecordsStatistics/index.html @@ -0,0 +1,5555 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + RollbackRecordsStatistics - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

RollbackRecordsStatistics

+ +

Rollback Records Statistics

+

Clicking on "Rollback Records Statistics" under the "Statistics Report" menu allows you to query all the rolled-back tasks. This page includes a search button, search criteria fields, and an export button. Each rolled-back task entry includes information such as task name, job name, archive type, execution start and end time, execution status, error message, progress percentage, archived item count, archive path, executed SQL, and rollback status.

+

image-20230621112909511

+ +

The green button in the image above is the search button. On the left side, there are three criteria fields. After filling in the filtering criteria, click "Search" to filter out rollback records that match the criteria.

+
Export
+

The yellow button is the export button, which allows you to export the rollback records statistics information as an xlsx file.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/UserManual/StatisticalReports/SystemAccessStatistics/index.html b/docs/whalealData/UserManual/StatisticalReports/SystemAccessStatistics/index.html new file mode 100644 index 00000000..ca9c872e --- /dev/null +++ b/docs/whalealData/UserManual/StatisticalReports/SystemAccessStatistics/index.html @@ -0,0 +1,5596 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + SystemAccessStatistics - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

SystemAccessStatistics

+ +

System Access Statistics

+

Clicking on "System Access Statistics" under the "Statistics Report" menu allows you to query all the businesses that have accessed this platform. This page includes a search button, search criteria fields, a clear criteria button, and an export button. Each entry represents a business that has accessed the platform, and it includes information such as business name, data source type, target source type, and access time.

+

image-20230621112420051

+ +

The green button in the image above is the search button. On the left side, there are two criteria fields. After filling in the filtering criteria, click "Search" to filter out access information that matches the criteria.

+
Clear
+

The black button is the clear button, which clears the criteria fields.

+
Export
+

The yellow button is the export button, allowing you to export the system access statistics information as an xlsx file.

+
Details
+

Clicking on the "Details" link for each access record provides more detailed information about that access, including the business name, access type, source and target types, and access time.

+

image-20230621112710472

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/UserManual/StatisticalReports/TableJobExecutionStatistics/index.html b/docs/whalealData/UserManual/StatisticalReports/TableJobExecutionStatistics/index.html new file mode 100644 index 00000000..2251eef7 --- /dev/null +++ b/docs/whalealData/UserManual/StatisticalReports/TableJobExecutionStatistics/index.html @@ -0,0 +1,5577 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + TableJobExecutionStatistics - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

TableJobExecutionStatistics

+ +

Table Job Execution Statistics

+

Clicking on "Table Job Execution Statistics" under the "Statistics Report" menu allows you to query the archival information for all table jobs. This page includes a search button, search criteria fields, a clear criteria button, and an export button. Each entry represents a table job execution and includes information such as job name, table name, data source type, target source type, execution start and end times, execution status, exception errors, progress percentage, number of archived rows, archival path, executed SQL, and rollback status.

+

image-20230621110252096

+ +

The green button in the image above is the search button. On the left side, there are two criteria fields. After filling in the filtering criteria, click "Search" to filter out job executions that match the criteria.

+
Clear
+

The black button is the clear button, which clears the criteria fields.

+
Export
+

The yellow button is the export button, allowing you to export the table job execution statistics information as an xlsx file.

+

These features help you keep track of and analyze the execution status and details of various table jobs in the system.

+

Please note that due to the screenshot's resolution, some text may be difficult to read.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/UserManual/SystemManagement/ErrorLog/index.html b/docs/whalealData/UserManual/SystemManagement/ErrorLog/index.html new file mode 100644 index 00000000..07850e5f --- /dev/null +++ b/docs/whalealData/UserManual/SystemManagement/ErrorLog/index.html @@ -0,0 +1,5535 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + ErrorLog - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

ErrorLog

+ +

Error Logs

+ +

Clicking on "System Operation Logs" under the "System Management" menu allows administrators to view error logs related to user actions on the platform. The page displays error logs generated from user actions. The green button is the search button. On the left side, there is a criteria field. Fill in the filtering criteria in the field, and then click "Search" to filter out the desired error logs. By clicking the "View Details" button on the right, you can view the details of the error and its causes.

+

image-20230621105907961

+

These error logs help administrators identify and address issues or unexpected behaviors in the system, providing insights into the activities that led to errors.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/UserManual/SystemManagement/MenuManagement/index.html b/docs/whalealData/UserManual/SystemManagement/MenuManagement/index.html new file mode 100644 index 00000000..20b1efe1 --- /dev/null +++ b/docs/whalealData/UserManual/SystemManagement/MenuManagement/index.html @@ -0,0 +1,5597 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + MenuManagement - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

MenuManagement

+ + +

Clicking on "Menu Management" under the "System Management" menu allows administrators to manage the menus within the platform. This page is visible only to administrator users. On this page, you can view information about all the menus, as well as perform actions such as searching, adding, modifying, and deleting menus.

+

image-20230620183823892

+ +

The green button at the top is the search button. On the left side, there are two criteria fields. Fill in the filtering criteria in these fields, and then click "Search" to filter the desired menus.

+
Add
+

The blue button is the "Add" button. Clicking it will bring up a form, as shown in the image below. In this form, you can customize the menu type, menu icon, external link menu, menu visibility, menu title, route address, menu sorting, parent directory, and more. Once you've configured the menu, click "Confirm" to save it.

+

image-20230620184102620

+
Edit
+

Clicking the blue pencil icon next to a menu's name will bring up an edit form where you can modify the menu's configuration according to your needs. Once you're done, click "Confirm" to save the changes.

+

image-20230620184307490

+
Delete
+

The delete button for menus is not available, likely to prevent accidental deletion. Instead, menus can be hidden based on requirements.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/UserManual/SystemManagement/OperationLog/index.html b/docs/whalealData/UserManual/SystemManagement/OperationLog/index.html new file mode 100644 index 00000000..1fac2e6e --- /dev/null +++ b/docs/whalealData/UserManual/SystemManagement/OperationLog/index.html @@ -0,0 +1,5534 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + OperationLog - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

OperationLog

+ +

Operation Logs

+ +

Clicking on "Operation Logs" under the "System Management" menu, which is visible only to administrator users, will take you to a page displaying operation logs of platform users. The green button is the search button. On the left side, there are criteria fields. Fill in the filtering criteria in these fields, and then click "Search" to filter the desired operation logs.

+

image-20230621105713881

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/UserManual/SystemManagement/RoleManagement/index.html b/docs/whalealData/UserManual/SystemManagement/RoleManagement/index.html new file mode 100644 index 00000000..7ec292f1 --- /dev/null +++ b/docs/whalealData/UserManual/SystemManagement/RoleManagement/index.html @@ -0,0 +1,5619 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + RoleManagement - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

RoleManagement

+ +

Role Management

+

Clicking on "Role Management" under the "System Management" menu, which is visible only to administrator users, will take you to a page where you can view information about all role permissions categories in the platform.

+

image-20230620182513207

+ +

The green button at the top is the search button. On the left side, there are two criteria fields. Fill in these fields and click "Search" to filter the desired role list.

+
Add
+

The blue button is the add button. Clicking the "Add" button will open the interface as shown in the image. Fill in the role name and role permissions to add role information.

+

image-20230620182736949

+
Edit
+

Clicking the blue pencil icon on the right side of a role opens the edit role dialog. You can modify the role's name, permissions, or add a description. Click "Save" when done.

+

image-20230620182754102

+
Delete
+

The red button on the right side is the delete button. Clicking the delete button will prompt whether you are sure you want to delete the role. If you confirm deletion, click "OK"; if not, click "Cancel."

+

image-20230620182902204

+ +

Clicking on a role reveals the menus associated with that role on the right side. Check and assign menus according to the actual permissions and menus the role should have. Save the data to bind the menus that the role can operate with.

+

image-20230620182949722

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/UserManual/SystemManagement/SystemSettings/index.html b/docs/whalealData/UserManual/SystemManagement/SystemSettings/index.html new file mode 100644 index 00000000..7040e083 --- /dev/null +++ b/docs/whalealData/UserManual/SystemManagement/SystemSettings/index.html @@ -0,0 +1,5503 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + SystemSettings - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

SystemSettings

+ +
SMTP Email Settings
+

Clicking on "System Settings" under the "System Management" menu, which is visible only to administrator users, will take you to a page where SMTP configuration is displayed. After filling in the basic configuration, click the "Test Connection" button. If the test is successful and you receive a test email, a "Save" button will appear. Once the SMTP configuration is saved, alert emails for task configurations will be sent from the email address configured here.

+

image-20230621105006375

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/UserManual/SystemManagement/UserManagement/index.html b/docs/whalealData/UserManual/SystemManagement/UserManagement/index.html new file mode 100644 index 00000000..34a52aad --- /dev/null +++ b/docs/whalealData/UserManual/SystemManagement/UserManagement/index.html @@ -0,0 +1,5660 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + UserManagement - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

UserManagement

+ +

User Management

+

Clicking on "User Management" under the "System Management" menu, visible only to administrator users, will take you to a page displaying all user information on the platform, including database permissions and bindings.

+

image-20230620180214867

+ +

The green button is the search button. Fill in the left two search boxes with relevant criteria and click the search button to filter the desired user information.

+
Add User
+

The second blue button is the "Add" button. Clicking on it will bring up the interface shown in the image above. Fill in the username, email, phone number, and password to add a new user. In the "Role" section, you can select the system permissions for the user, as well as the user's status (enabled or disabled).

+

image-20230620180556039

+
Export Users
+

The third yellow button is the "Export Users" button. Clicking on it will generate an xlsx file containing information for all users. You can customize the file name.

+
Modify User Details
+

Clicking on the pencil icon on the right side of a user's row will allow you to modify user information. An edit page will pop up with editable fields for all user details except for the password. Modify the information as needed and click "Confirm" to save the changes.

+

image-20230620181013810

+
Change Password
+

Clicking on the second password icon on the right side of a user's row will allow you to change the user's password. A form will appear where you can enter the new password. After entering the new password, click "Confirm" to save the changes.

+

image-20230620181322395

+
Delete User
+

Clicking on the third red button on the right side of a user's row will prompt a confirmation message asking if you want to delete the user. If you confirm deletion, click "Confirm." If you don't want to delete the user, click "Cancel."

+

image-20230620181434250

+
Database Permissions (Data Sources, Target Sources, File Sources, S3)
+

Clicking on a user's row will display the database permissions assigned to that user, including data sources, target sources, file sources, and S3. Check the sources that the user should have access to, then click "Save."

+

image-20230620181537676

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/UserManual/TaskManagement/ColdTaskMonitoring/index.html b/docs/whalealData/UserManual/TaskManagement/ColdTaskMonitoring/index.html new file mode 100644 index 00000000..1841bdc7 --- /dev/null +++ b/docs/whalealData/UserManual/TaskManagement/ColdTaskMonitoring/index.html @@ -0,0 +1,5617 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + ColdTaskMonitoring - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

ColdTaskMonitoring

+ +

Task Monitoring (Cold)

+
Task Status
+

Clicking on "Task Monitoring (Cold)" under the "Task Management" menu will display information about the execution status of cold tasks. This page includes information about completed tasks, ongoing tasks, and tasks with exceptions. Each search button is associated with a set of search criteria, allowing you to filter and display tasks efficiently.

+

image-20230620150611237

+
Completed Tasks
+

Clicking on "Completed Tasks" will display information about tasks that have been successfully completed. This page includes details such as the execution strategy, start and end times, duration, execution status, progress percentage, archived items, archive path, source table data status, and executed SQL statements. The page also features four buttons: search, modify source table data status, manually delete source table data, and refresh.

+

image-20230620150913972

+

Search

+

The green button at the top is the search button. Enter criteria in the provided search boxes and click the search button to display filtered completed tasks.

+

Modify Source Table Data Status

+

After synchronization is completed, if source table data has been manually deleted, you can click the yellow button to mark the source table data as processed.

+

Manually Delete Source Table Data

+

The red button allows you to manually delete source table data. If the table job configuration does not include automatic deletion and you want to manually delete the source data after synchronization, you can use this button.

+

Refresh

+

The progress percentage of a task is updated every 3 seconds. Clicking the refresh button will update the progress bar and task status.

+
Ongoing Tasks
+

Clicking on "Ongoing Tasks" will display information about tasks that are currently in progress. This page includes details such as the execution strategy, start and end times, duration, execution status, progress percentage, archived items, and executed SQL statements. The page features three buttons: search, terminate task, and task status validation.

+

image-20230620151106771

+

Search

+

The green button is the search button. Fill in the provided search boxes as needed, and then click the search button to display all ongoing tasks that match the criteria.

+

Terminate Task

+

The red button allows you to terminate an ongoing task. After clicking this button, the task will be terminated and will appear in the list of tasks with exceptions. If the termination is successful, the task will also be marked as completed.

+

Task Status Validation

+

Each task can contain multiple table jobs. When one table job is completed, the next one starts. If the task status does not update after a table job has been completed, you can click the "Task Status Validation" button to update the task status.

+
Exception Tasks
+

Clicking on "Exception Tasks" will display information about tasks that encountered exceptions. This page includes details such as the execution strategy, start and end times, duration, execution status, exception details, progress percentage, archived items, archive path, executed SQL statements, and rollback status. This page features three buttons: search, rollback, and re-execute.

+

image-20230620160849270

+

Search

+

The green button at the top is the search button. Fill in the provided search boxes with criteria and click the search button to display filtered exception tasks.

+

Rollback

+

Each exception task has a rollback button. Clicking on this button will initiate a rollback of the exception data synchronized by the subtask. If the rollback button of the parent task is clicked, all subtasks under that parent task will be rolled back.

+

Re-execute

+

Each subtask of an exception task has a re-execute button. Clicking this button will generate a new parent task associated with the exception task. The exception task will be linked to the new parent task. After the rollback of the exception data is completed, the task will appear in the ongoing tasks list, and you can re-execute it.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/UserManual/TaskManagement/S3TaskMonitoring/index.html b/docs/whalealData/UserManual/TaskManagement/S3TaskMonitoring/index.html new file mode 100644 index 00000000..513da333 --- /dev/null +++ b/docs/whalealData/UserManual/TaskManagement/S3TaskMonitoring/index.html @@ -0,0 +1,5617 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + S3TaskMonitoring - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

S3TaskMonitoring

+ +

Task Monitoring (S3)

+

Clicking on "Task Monitoring (S3)" under the "Task Management" menu will display information about the execution status of S3 tasks. This page includes information about completed tasks, ongoing tasks, and tasks with exceptions. Each search button is associated with a set of search criteria, allowing you to filter and display tasks efficiently.

+

image-20230620161115219

+
Task Status
+
Completed Tasks
+

Clicking on "Completed Tasks" will display information about tasks that have been successfully completed. This page includes details such as the execution strategy, start and end times, duration, execution status, progress percentage, archived items, source table data status, and executed SQL statements. The page also features four buttons: search, modify source table data status, manually delete source table data, and refresh.

+

image-20230620164039744

+

Search

+

The green button at the top is the search button. Enter criteria in the provided search boxes and click the search button to display filtered completed tasks.

+

Modify Source Table Data Status

+

After synchronization is completed, if source table data has been manually deleted, you can click the yellow button to mark the source table data as processed.

+

Manually Delete Source Table Data

+

The red button allows you to manually delete source table data. If the table job configuration does not include automatic deletion and you want to manually delete the source data after synchronization, you can use this button.

+

Refresh

+

The progress percentage of a task is updated every 3 seconds. Clicking the refresh button will update the progress bar and task status.

+
Ongoing Tasks
+

Clicking on "Ongoing Tasks" will display information about tasks that are currently in progress. This page includes details such as the execution strategy, start and end times, duration, execution status, progress percentage, archived items, and executed SQL statements. The page features three buttons: search, terminate task, and task status validation.

+

image-20230620164238193

+

Search

+

The green button is the search button. Fill in the provided search boxes as needed, and then click the search button to display all ongoing tasks that match the criteria.

+

Terminate Task

+

The red button allows you to terminate an ongoing task. After clicking this button, the task will be terminated and will appear in the list of tasks with exceptions. If the termination is successful, the task will also be marked as completed.

+

Task Status Validation

+

Each task can contain multiple table jobs. When one table job is completed, the next one starts. If the task status does not update after a table job has been completed, you can click the "Task Status Validation" button to update the task status.

+
Exception Tasks
+

Clicking on "Exception Tasks" will display information about tasks that encountered exceptions. This page includes details such as the execution strategy, start and end times, duration, execution status, exception details, progress percentage, archived items, executed SQL statements, and rollback status. This page features three buttons: search, rollback, and re-execute.

+

image-20230620164349940

+

Search

+

The green button at the top is the search button. Fill in the provided search boxes with criteria and click the search button to display filtered exception tasks.

+

Rollback

+

Each exception task has a rollback button. Clicking on this button will initiate a rollback of the exception data synchronized by the subtask. If the rollback button of the parent task is clicked, all subtasks under that parent task will be rolled back.

+

Re-execute

+

Each subtask of an exception task has a re-execute button. Clicking this button will generate a new parent task associated with the exception task. The exception task will be linked to the new parent task. After the rollback of the exception data is completed, the task will appear in the ongoing tasks list, and you can re-execute it.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/UserManual/TaskManagement/TaskScheduling/index.html b/docs/whalealData/UserManual/TaskManagement/TaskScheduling/index.html new file mode 100644 index 00000000..9c54fa2c --- /dev/null +++ b/docs/whalealData/UserManual/TaskManagement/TaskScheduling/index.html @@ -0,0 +1,5595 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + TaskScheduling - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

TaskScheduling

+ +

Task Scheduling

+

Clicking on "Task Scheduling" under the "Task Management" menu will display a page that shows tasks that have been approved. The search button with associated search criteria boxes can be used to filter and display tasks based on the specified criteria. For individual tasks, you can modify their status, such as enabling, disabling, taking them offline, or executing them immediately.

+

image-20230620133351561

+
Enable Task
+

After a task has been disabled, it cannot be executed. You can enable a disabled task by clicking the "Enable Task" button on the right side of the task. After enabling the task, you can click the "Run Now" button to execute the task immediately or let it run automatically at the scheduled time.

+
Disable Task
+

You can click the gray button on the right side of a task to disable it. Once a task is disabled, it will not be executed. You need to enable the task again for it to resume normal operation.

+
Run Now
+

The third button on the right side of a task allows you to execute it immediately. For full synchronization tasks, clicking this button will initiate the synchronization immediately. For incremental synchronization tasks, clicking "Run Now" will execute the next scheduled task that has not yet reached its execution time.

+
Take Task Offline
+

The rightmost button, "Take Task Offline," allows you to take a task offline. Clicking this button will prompt a confirmation dialog to confirm the offline operation. Once a task is taken offline, it cannot be restored, and the task will become unavailable. If you want to proceed with taking the task offline, click "Take Offline." If you want to cancel, click "Cancel."

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/UserManual/TaskManagement/WarmTaskMonitoring/index.html b/docs/whalealData/UserManual/TaskManagement/WarmTaskMonitoring/index.html new file mode 100644 index 00000000..51a60fa3 --- /dev/null +++ b/docs/whalealData/UserManual/TaskManagement/WarmTaskMonitoring/index.html @@ -0,0 +1,5595 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + WarmTaskMonitoring - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

WarmTaskMonitoring

+ +

Task Monitoring (Warm)

+

Clicking on "Task Monitoring (Warm)" under the "Task Management" menu will display a page that shows the execution status of warm tasks. The page includes information about completed tasks, tasks in progress, and exceptional tasks. Each search button is associated with condition boxes that allow you to filter and display tasks based on specified criteria.

+

image-20230620140353683

+
Completed Tasks
+

Clicking on completed tasks will display information about tasks that have been successfully completed. The information includes execution strategy, start and end times, execution duration, execution status, progress percentage, archived records count, source data status, and executed SQL statements. There are four buttons at the top: Search, Modify Source Data Status, Manually Delete Source Data, and Refresh.

+

Search

+

The green button at the top is the search button. By entering conditions in the provided boxes and clicking on the search button, you can filter and display completed tasks that match the criteria.

+

Modify Source Data Status

+

After a synchronization is completed and if source data has been manually deleted, you can click the yellow button to modify the source data status to "processed."

+

Manually Delete Source Data

+

The red button allows you to manually delete source data. If automatic deletion is not configured in the table job settings, you can manually delete the source data from the database. Alternatively, you can click the "Manually Delete Source Data" button after selecting a task.

+

Refresh

+

The progress percentage of a task is updated every 3 seconds. As a result, the progress bar display might not be real-time. Clicking the refresh button updates the progress bar and some task statuses.

+
Tasks in Progress
+

Clicking on tasks in progress will display information about tasks that are currently being executed and archived. The information includes execution strategy, start and end times, execution duration, execution status, progress percentage, archived records count, executed SQL statements. There are three buttons: Search, Terminate Task, and Verify Task Status.

+

Search

+

The green button is the search button. Enter conditions in the provided boxes and click the search button to filter and display tasks in progress that match the criteria.

+

Terminate Task

+

The red button allows you to terminate a task. After selecting a task and clicking the "Terminate Task" button, the task will be terminated. The task will then appear in the exceptional tasks section if it was not completed normally.

+

Verify Task Status

+

A task can include multiple table jobs. When one table job is completed, the next one starts. If a task's status does not update promptly after a table job is completed, you can click the "Verify Task Status" button to update the task's status.

+
Exceptional Tasks
+

Clicking on exceptional tasks will display information about tasks that encountered exceptions. The information includes execution strategy, start and end times, execution duration, execution status, error details, progress percentage, archived records count, executed SQL statements, and rollback status. There are three buttons: Search, Rollback, and Re-execute.

+

Search

+

The green button is the search button. Enter conditions in the provided boxes and click the search button to filter and display exceptional tasks that match the criteria.

+

Rollback

+

Each exceptional task has a rollback button. Clicking the rollback button for a sub-task will roll back the exceptional data that was synchronized. Clicking the rollback button for a parent task will roll back all sub-tasks under that parent task.

+

Re-execute

+

Sub-tasks under an exceptional task have a re-execute button. Clicking the re-execute button will generate a new parent task. The exceptional task and the new parent task will be linked. After rolling back the exceptional data, both the exceptional task and the new parent task will appear in the "Tasks in Progress" section for re-execution.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/index.html b/docs/whalealData/index.html new file mode 100644 index 00000000..3bac704b --- /dev/null +++ b/docs/whalealData/index.html @@ -0,0 +1,5466 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + Whaleal-data Introduction - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

Whaleal-data Introduction

+

Whaleal-data is an archiving platform designed for data archiving. It supports three types of archiving: cold, warm, and S3. For warm data archiving, it supports synchronizing data from MYSQL, Oracle, DB2, and MongoDB to MongoDB. It also supports synchronizing data from MYSQL to MYSQL. Cold data archiving allows archiving MongoDB data to disk files. S3 archiving supports uploading MongoDB Gridfs data to target S3 storage. The platform's homepage displays task execution statistics, archive capacity statistics, table job statistics, business connections, total archive capacity, and user operations. The platform enables independent configuration of data sources and target sources. Users can choose a source to sync data from and a target to sync data to within a table job. After configuring table jobs, multiple table jobs can be configured within a single task configuration. Once tasks are created, they can be managed in the task scheduling section by enabling, disabling, immediately executing, or taking tasks offline. After immediate execution, the corresponding task execution details can be viewed in the task monitoring section. Administrator users have access to various operational details.

+

Platform Architecture Diagram

+

Platform Architecture Diagram.png

+

Software Structure Diagram

+

Software Structure Diagram.png

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/use cases/AbnormalTaskFeedback/index.html b/docs/whalealData/use cases/AbnormalTaskFeedback/index.html new file mode 100644 index 00000000..d2c2c4e9 --- /dev/null +++ b/docs/whalealData/use cases/AbnormalTaskFeedback/index.html @@ -0,0 +1,5506 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + AbnormalTaskFeedback - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

AbnormalTaskFeedback

+ +

Handling Exceptional Tasks

+

When encountering exceptions during cold data archiving, you can follow these steps to address the issue:

+
    +
  1. Navigate to the "Archive Management (Cold Data)" menu and select "Log Query (Cold)".
    2. +
  2. This page will display the archived cold data tasks. Click on "File Rewriting" to perform a rollback of the archived files into the database.
    3. +
+

File Rewriting

+

This process allows you to manage and recover from any anomalies that might occur during the cold data archiving process.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/use cases/AddColdDataFullLoadJob/index.html b/docs/whalealData/use cases/AddColdDataFullLoadJob/index.html new file mode 100644 index 00000000..d14c6bbb --- /dev/null +++ b/docs/whalealData/use cases/AddColdDataFullLoadJob/index.html @@ -0,0 +1,5510 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + AddColdDataFullLoadJob - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

AddColdDataFullLoadJob

+ +

Adding a Cold Data Full Job

+

To set up a full job for cold data archiving, follow these steps:

+
    +
  1. Click on the "Configuration Management" menu and select "Table Job Configuration".
    2. +
  2. In the "Cold Data Table Job" page, click on the blue "Add" button to open the form.
    3. +
  3. Fill out the form by selecting the data source table and file source for synchronization. Choose the archiving mode as "Full Update".
    4. +
  4. Note that cold data archiving is applicable only to MongoDB data.
    5. +
  5. The table job comes with consistency validation. Select "Yes" and specify the required validation percentage. After synchronization, the platform will perform consistency validation on the synchronized data.
    6. +
  6. Choose the data handling method, either manual deletion or system deletion. This feature allows you to delete the source table after synchronization is completed.
    7. +
+

Cold Data Full Job

+

Following these steps will enable you to configure a full job for cold data archiving, ensuring efficient and accurate data synchronization and archiving.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/use cases/AddColdDataIncrementalJob/index.html b/docs/whalealData/use cases/AddColdDataIncrementalJob/index.html new file mode 100644 index 00000000..c4cf25df --- /dev/null +++ b/docs/whalealData/use cases/AddColdDataIncrementalJob/index.html @@ -0,0 +1,5512 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + AddColdDataIncrementalJob - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

AddColdDataIncrementalJob

+ +

Adding a Cold Data Incremental Job

+

To set up an incremental job for cold data archiving, follow these steps:

+
    +
  1. Click on the "Configuration Management" menu and select "Table Job Configuration".
    2. +
  2. In the "Cold Data Table Job" page, click on the blue "Add" button to open the form.
    3. +
  3. Fill out the form by selecting the data source table and file source for synchronization. Choose the archiving mode as "Incremental Update".
    4. +
  4. Note that cold data archiving is applicable only to MongoDB data.
    5. +
  5. For the incremental update mode, you need to fill in the SQL configuration. Click the blue "Custom SQL" button to open the table shown in the second image. Select the completion conditions and click "Save" to generate the SQL.
    6. +
  6. The table job comes with consistency validation. Select "Yes" and specify the required validation percentage. After synchronization, the platform will perform consistency validation on the synchronized data.
    7. +
  7. Choose the data handling method, either manual deletion or system deletion. This feature deletes the source table based on the configured batch.
    8. +
+

Cold Data Incremental Job

+

Custom SQL Configuration

+

By following these steps, you can configure an incremental job for cold data archiving, enabling efficient and accurate synchronization of MongoDB data for archiving purposes.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/use cases/AddDataSource/index.html b/docs/whalealData/use cases/AddDataSource/index.html new file mode 100644 index 00000000..c5656378 --- /dev/null +++ b/docs/whalealData/use cases/AddDataSource/index.html @@ -0,0 +1,5510 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + AddDataSource - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

AddDataSource

+ +

Adding a Data Source

+

To add a new data source to the platform, follow these steps:

+
    +
  1. Click on the "Configuration Management" menu and select "Data Source Management".
    2. +
  2. Click the blue "Add" button to open the form.
    3. +
  3. Fill in the basic information for the new data source and click "Test". If you see a "Test Passed" message above, click "Save" to successfully add the data source. If the test fails, double-check the provided information for accuracy.
    4. +
  4. After adding the data source, go to the "System Management" menu and select "User Management". Click on the user you want to bind the data source to.
    5. +
  5. In the user details, select the added data source from the list and click "Save". This user will now have access to the newly added data source.
    6. +
+

Adding a Data Source

+

Binding Data Source to User

+

By following these steps, you can easily add and configure new data sources, allowing users to access and utilize these sources for various operations within the Whaleal Data platform.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/use cases/AddDestinationSource/index.html b/docs/whalealData/use cases/AddDestinationSource/index.html new file mode 100644 index 00000000..870d851f --- /dev/null +++ b/docs/whalealData/use cases/AddDestinationSource/index.html @@ -0,0 +1,5510 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + AddDestinationSource - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

AddDestinationSource

+ +

Adding a Target Source

+

To add a new target source to the platform, follow these steps:

+
    +
  1. Click on the "Configuration Management" menu and select "Target Source Management".
    2. +
  2. Click the blue "Add" button to open the form.
    3. +
  3. Fill in the basic information for the new target source and click "Test". If you see a "Test Passed" message above, click "Save" to successfully add the target source. If the test fails, double-check the provided information for accuracy.
    4. +
  4. After adding the target source, go to the "System Management" menu and select "User Management". Click on the user you want to bind the target source to.
    5. +
  5. In the user details, select the added target source from the list and click "Save". This user will now have access to the newly added target source.
    6. +
+

Adding a Target Source

+

Binding Target Source to User

+

By following these steps, you can add and configure new target sources, allowing users to utilize these sources as destinations for data operations within the Whaleal Data platform.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/use cases/AddS3FullLoadJob/index.html b/docs/whalealData/use cases/AddS3FullLoadJob/index.html new file mode 100644 index 00000000..e1fae4a2 --- /dev/null +++ b/docs/whalealData/use cases/AddS3FullLoadJob/index.html @@ -0,0 +1,5511 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + AddS3FullLoadJob - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

AddS3FullLoadJob

+ +

Adding an S3 Full Load Job

+

To add a new S3 full load job to the platform, follow these steps:

+
    +
  1. Click on the "Configuration Management" menu and select "Table Job Configuration".
    2. +
  2. In the S3 Table Job page, click the blue "Add" button to open the form.
    3. +
  3. Choose the data source database table and the target S3 bucket you want to synchronize. Ensure that the source endpoint is MongoDB's Gridfs data as the source for S3 synchronization.
    4. +
  4. Select "Full Load" as the archive mode.
    5. +
  5. The table job also includes data consistency verification. If you choose to enable it, you can set the required verification percentage. After the synchronization, the platform will perform data consistency checks on the synchronized data.
    6. +
  6. Since S3 has the characteristic that files with the same name will overwrite the existing files, you can choose from synchronization modes like "Replace without Handling", "Replace with Newest Files", or "ID + Filename" mode.
    7. +
  7. Choose a data processing method, either manual deletion or automatic deletion, after synchronization. The data source table will be deleted according to your choice after synchronization is completed.
    8. +
+

Adding an S3 Full Load Job

+

By following these steps, you can create an S3 full load job that synchronizes data from a MongoDB Gridfs data source to a target S3 bucket. This allows for efficient management and synchronization of data between different storage systems within the Whaleal Data platform.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/use cases/AddS3IncrementalJob/index.html b/docs/whalealData/use cases/AddS3IncrementalJob/index.html new file mode 100644 index 00000000..46f19c34 --- /dev/null +++ b/docs/whalealData/use cases/AddS3IncrementalJob/index.html @@ -0,0 +1,5514 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + AddS3IncrementalJob - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

AddS3IncrementalJob

+ +

Adding an S3 Incremental Load Job

+

To add a new S3 incremental load job to the platform, follow these steps:

+
    +
  1. Click on the "Configuration Management" menu and select "Table Job Configuration".
    2. +
  2. In the S3 Table Job page, click the blue "Add" button to open the form.
    3. +
  3. Choose the data source database table and the target S3 bucket you want to synchronize. Ensure that the source endpoint is MongoDB's Gridfs data as the source for S3 synchronization.
    4. +
  4. Select "Incremental Load" as the archive mode.
    5. +
  5. You'll need to provide SQL configuration since you're choosing incremental mode. Click the blue "Custom SQL" button to open the form for defining the SQL conditions.
    6. +
  6. Set up the conditions for the incremental synchronization SQL in the form. After defining the conditions, click "Save" to generate the SQL.
    7. +
  7. The table job also includes data consistency verification. If you choose to enable it, you can set the required verification percentage. After the synchronization, the platform will perform data consistency checks on the synchronized data.
    8. +
  8. Since S3 has the characteristic that files with the same name will overwrite the existing files, you can choose from synchronization modes like "Replace without Handling", "Replace with Newest Files", or "ID + Filename" mode.
    9. +
  9. Choose a data processing method, either manual deletion or automatic deletion, after synchronization. The data source table will be deleted according to your choice after synchronization is completed.
    10. +
+

Adding an S3 Incremental Load Job

+

Custom SQL for Incremental Load

+

By following these steps, you can create an S3 incremental load job that synchronizes data from a MongoDB Gridfs data source to a target S3 bucket using incremental synchronization based on defined SQL conditions. This allows for efficient and selective data synchronization within the Whaleal Data platform.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/use cases/AddWarmDataFullLoadJob/index.html b/docs/whalealData/use cases/AddWarmDataFullLoadJob/index.html new file mode 100644 index 00000000..ae974761 --- /dev/null +++ b/docs/whalealData/use cases/AddWarmDataFullLoadJob/index.html @@ -0,0 +1,5511 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + AddWarmDataFullLoadJob - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

AddWarmDataFullLoadJob

+ +

Adding a Warm Data Full Load Job

+

To add a new warm data full load job to the platform, follow these steps:

+
    +
  1. Click on the "Configuration Management" menu and select "Table Job Configuration".
    2. +
  2. In the Warm Data Table Job page, click the blue "Add" button to open the form.
    3. +
  3. Choose the data source database table and the target destination database table that you want to synchronize.
    4. +
  4. Select "Full Load" as the archive mode.
    5. +
  5. The warm data table job also includes data consistency verification. If you choose to enable it, you can set the required verification percentage. After the synchronization, the platform will perform data consistency checks on the synchronized data.
    6. +
  6. For MySQL-related jobs, you can choose the isolation level for synchronization. For MongoDB to MongoDB synchronization, you can choose whether to sync Gridfs. If you choose "No," the platform will only synchronize regular documents.
    7. +
  7. Choose a data processing method, either manual deletion or automatic deletion, after synchronization. The data source table will be deleted according to your choice after synchronization is completed.
    8. +
+

Adding a Warm Data Full Load Job

+

By following these steps, you can create a warm data full load job that synchronizes data from a data source database table to a target destination database table using full load mode. This allows for comprehensive synchronization of data within the Whaleal Data platform.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/use cases/AddWarmDataIncrementalJob/index.html b/docs/whalealData/use cases/AddWarmDataIncrementalJob/index.html new file mode 100644 index 00000000..794095f7 --- /dev/null +++ b/docs/whalealData/use cases/AddWarmDataIncrementalJob/index.html @@ -0,0 +1,5512 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + AddWarmDataIncrementalJob - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

AddWarmDataIncrementalJob

+ +

Adding a Warm Data Incremental Load Job

+

To add a new warm data incremental load job to the platform, follow these steps:

+
    +
  1. Click on the "Configuration Management" menu and select "Table Job Configuration".
    2. +
  2. In the Warm Data Table Job page, click the blue "Add" button to open the form.
    3. +
  3. Choose the data source database table and the target destination database table that you want to synchronize.
    4. +
  4. Select "Incremental Load" as the archive mode.
    5. +
  5. If you choose the incremental load mode, you need to fill in the SQL configuration. Click the blue "Custom SQL" button to open the form where you can select the completion conditions for the SQL. Click "Save" to generate the SQL.
    6. +
  6. The warm data table job also includes data consistency verification. If you choose to enable it, you can set the required verification percentage. After the synchronization, the platform will perform data consistency checks on the synchronized data.
    7. +
  7. For MySQL-related jobs, you can choose the isolation level for synchronization. For MongoDB to MongoDB synchronization, you can choose whether to sync Gridfs. If you choose "No," the platform will only synchronize regular documents.
    8. +
  8. Choose a data processing method, either manual deletion or automatic deletion, after synchronization. The data source table will be deleted according to your choice after synchronization is completed.
    9. +
+

Adding a Warm Data Incremental Load Job

+

By following these steps, you can create a warm data incremental load job that synchronizes data from a data source database table to a target destination database table using incremental load mode. This allows for continuous synchronization of data within the Whaleal Data platform.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/use cases/ColdDataFullTextSearch/index.html b/docs/whalealData/use cases/ColdDataFullTextSearch/index.html new file mode 100644 index 00000000..e1807c21 --- /dev/null +++ b/docs/whalealData/use cases/ColdDataFullTextSearch/index.html @@ -0,0 +1,5559 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + ColdDataFullTextSearch - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

ColdDataFullTextSearch

+ +

Cold Data Full-Text Search

+

To perform a full-text search on archived cold data, follow these steps:

+
    +
  1. Click on the "Archive Management" menu and select "File Full-Text Search". This will display all file full-text search log information.
    2. +
  2. Click the "Create Search Task" button to create a file full-text search task. The system will execute a global search on the files. For the files that match the search criteria, you can perform a file rollback operation.
    3. +
+

Cold Data Full-Text Search

+
Searching
+

The green button on the top is the search button. The left-side filter options can be filled in as needed. After filling in the conditions, click the search button to filter the desired search content.

+
Creating a Search Task
+

Click the blue button to create a search task. This will open a form where you can enter the necessary details. After filling in the required information, click "Confirm".

+

Creating a Search Task

+

By following these steps, you can search for archived cold data using the full-text search feature in the Whaleal Data platform. You can filter and retrieve specific files based on your search criteria and even perform file rollback operations as needed.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/use cases/ColdDataWriteBack/index.html b/docs/whalealData/use cases/ColdDataWriteBack/index.html new file mode 100644 index 00000000..e66c3950 --- /dev/null +++ b/docs/whalealData/use cases/ColdDataWriteBack/index.html @@ -0,0 +1,5511 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + ColdDataWriteBack - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

ColdDataWriteBack

+ +

Cold Data Rollback

+

If there are issues with cold data archiving, you can perform a rollback operation. Here's how:

+
    +
  1. Go to the "Archive Management" menu and select "Log Query (Cold)" under "Cold Data". This page displays the archived cold data tasks.
    2. +
  2. Click on the "File Rollback" button to perform a rollback operation for a specific task. This action will roll back the archived file into the database.
    3. +
+

Cold Data Rollback

+

After clicking the task, you will see a yellow "File Rollback" button next to it. Click this button to initiate the rollback process.

+

Creating Rollback Task

+

Creating a Rollback Task

+

Clicking the "File Rollback" button will open a form. Fill in the required information and click "Confirm" to create the rollback task.

+

Creating Rollback Task Form

+

By following these steps, you can initiate a rollback operation for archived cold data files that need to be brought back into the database. This ensures that the data remains accessible and consistent within your system.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/use cases/ColdWorkDemo/index.html b/docs/whalealData/use cases/ColdWorkDemo/index.html new file mode 100644 index 00000000..df292e1c --- /dev/null +++ b/docs/whalealData/use cases/ColdWorkDemo/index.html @@ -0,0 +1,5778 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + ColdWorkDemo - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

ColdWorkDemo

+ +

Cold Job Demo

+

Here's a step-by-step demonstration of how to create and manage a cold data archiving job in the platform:

+

1. Add Data Source and File Source

+

Add Data Source

+
    +
  1. Go to the "Configuration Management" menu and select "Data Source Management".
    2. +
  2. Click the blue "Add" button and fill in the required information for the data source. Test the connection and save it if the test is successful.
    3. +
+

image-20230621132912082

+

Add File Source

+
    +
  1. Go to the "Configuration Management" menu and select "Target Source Management".
    2. +
  2. Click on "Cold Data File Source" and then click the blue "Add" button.
    3. +
  3. Fill in the necessary information related to cold data archiving and save the file source.
    4. +
+

image-20230619172709222

+

2. Assign Permissions

+

After adding the data source and file source, administrators need to assign database permissions in the "User Management" page under "System Management".

+

权限分配.png

+

3. Create a Cold Job

+
    +
  1. Go to the "Configuration Management" menu and select "Table Job Configuration".
    2. +
  2. Click the blue "Add" button in the "Cold Data Table Job" section.
    3. +
  3. Fill in the required information, including selecting the data source and file source.
    4. +
  4. Optionally, configure data consistency validation and choose the data handling method after synchronization.
    5. +
  5. Save the job configuration.
    6. +
+

新建冷作业.png

+

4. Create a Task

+
    +
  1. Click on the "Task Configuration" menu and then "Cold Data Task Configuration".
    2. +
  2. Click the blue "Add" button to create a new task configuration.
    3. +
  3. Fill in the necessary details according to your requirements.
    4. +
  4. Optionally, configure execution mode, task timeout, retry attempts, and notification settings.
    5. +
  5. Choose the notification strategy and add email addresses for alerts.
    6. +
  6. Save the task configuration.
    7. +
+

新增任务配置.png

+

Add Table Jobs to the Task

+
    +
  1. Click "Add Job" to associate table jobs with the created task.
    2. +
  2. Select the desired table jobs and click "Confirm" to add them to the task.
    3. +
+

添加表作业.png

+

Admin Approval

+
    +
  1. After configuring the task, an administrator can review and approve it if needed.
    2. +
  2. The task is ready for execution after approval.
    3. +
+

管理员审核.png

+

5. Task Scheduling

+
    +
  1. Navigate to the "Task Management" menu and select "Task Scheduling".
    2. +
  2. Monitor the status of the task in this section.
    3. +
  3. Click "Execute Now" to immediately execute the task.
    4. +
+

任务调度.png

+

6. Task Monitoring

+
    +
  1. In the task scheduling section, you can monitor the execution status of the task.
    2. +
  2. Visit the "Task Monitoring (Cold)" page to view the detailed execution status of the task.
    3. +
+

image-20230620150913972

+

By following these steps, you can successfully create, configure, and manage cold data archiving jobs in the platform. This allows you to archive data from various sources and monitor the execution of tasks for data consistency and reliability.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/use cases/CreateLoopTask/index.html b/docs/whalealData/use cases/CreateLoopTask/index.html new file mode 100644 index 00000000..a0bb5bc9 --- /dev/null +++ b/docs/whalealData/use cases/CreateLoopTask/index.html @@ -0,0 +1,5515 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + CreateLoopTask - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

CreateLoopTask

+ +

Creating Recurring Tasks

+

Creating a recurring task is similar to creating a one-time task. The key difference is that you need to set the execution interval for the recurring task, which determines when the task will be executed again in the future. Here's how you can create and manage recurring tasks:

+
    +
  1. Follow the same steps for creating a task configuration, just like you did for the one-time task.
    2. +
  2. In the task configuration, set the execution mode to "Recurring."
    3. +
  3. Specify the execution interval, which determines how often the task will be repeated.
    4. +
  4. Fill in the rest of the task configuration details and save the configuration.
    5. +
+

After configuring the recurring task, an administrator needs to review and approve it, just like with one-time tasks. Once the recurring task is approved, you can find it in the "Task Scheduling" section.

+

To execute a recurring task:

+
    +
  1. Navigate to the "Task Management" menu and select "Task Scheduling."
    2. +
  2. Locate the recurring task you want to execute.
    3. +
  3. Click the "Execute Now" button next to the task. This will trigger the immediate execution of the task according to the predefined execution interval.
    4. +
+

By creating recurring tasks, you can automate data archiving and synchronization at regular intervals, ensuring that your data remains up-to-date and consistent over time.

+

创建循环任务.png

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/use cases/CreateManualTask/index.html b/docs/whalealData/use cases/CreateManualTask/index.html new file mode 100644 index 00000000..dd370174 --- /dev/null +++ b/docs/whalealData/use cases/CreateManualTask/index.html @@ -0,0 +1,5514 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + CreateManualTask - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

CreateManualTask

+ +

Creating Manual Tasks

+

Creating a manual task follows the same steps as creating a one-time task. The key difference with manual tasks is that they don't require setting an execution time, as they are intended to be triggered manually when needed. Here's how you can create and manage manual tasks:

+
    +
  1. Begin by creating a task configuration, just as you did for the one-time task.
    2. +
  2. In the task configuration, set the execution mode to "Manual."
    3. +
  3. Complete the rest of the task configuration details and save the configuration.
    4. +
+

Once the manual task is configured, it will be available for execution by authorized users. An administrator needs to review and approve the manual task, just like with other tasks, before it can be executed.

+

To execute a manual task:

+
    +
  1. Navigate to the "Task Management" menu and select "Task Scheduling."
    2. +
  2. Locate the manual task you want to execute.
    3. +
  3. Click the "Execute Now" button next to the task. This will trigger the immediate execution of the task.
    4. +
+

Manual tasks are particularly useful when you want to perform specific data archiving or synchronization tasks on-demand, giving you full control over when these tasks are executed.

+

创建手工任务.png

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/use cases/CreateSingleTask/index.html b/docs/whalealData/use cases/CreateSingleTask/index.html new file mode 100644 index 00000000..d512bde8 --- /dev/null +++ b/docs/whalealData/use cases/CreateSingleTask/index.html @@ -0,0 +1,5515 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + CreateSingleTask - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

CreateSingleTask

+ +

Creating One-Time Tasks

+

To create a one-time task, follow these steps:

+
    +
  1. Navigate to the "Task Configuration" menu and select "Task Configuration."
    2. +
  2. Click the blue "New" button to open the task creation form.
    3. +
  3. In the task creation form (second image), select the task mode as "One-Time."
    4. +
  4. Choose a specific execution time using a Cron expression. Cron expressions allow you to define the exact date and time when the task should be executed.
    5. +
  5. Configure other settings as needed, such as execution mode, task timeout, and retry attempts.
    6. +
  6. Optionally, set up a notification strategy by adding email addresses for alerts. Notifications will be sent based on the chosen strategy after the task completes.
    7. +
  7. Click the "Add Job" button to attach a job (table job) to the task. In the job configuration form (third image), select the desired job(s) to be associated with this task.
    8. +
  8. Click "OK" or "Confirm" to save the task configuration.
    9. +
+

Please note that one-time tasks need to be reviewed and approved by an administrator before they can be executed. Once the task is approved, it will be scheduled for execution based on the specified time using the Cron expression.

+

One-time tasks are suitable for tasks that need to be executed at a specific point in time, such as data synchronization or archiving activities that are scheduled to happen once.

+

image-20230621142941634

+

image-20230621143119669

+

image-20230621143532776

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/use cases/ManuallyDeleteSourceData/index.html b/docs/whalealData/use cases/ManuallyDeleteSourceData/index.html new file mode 100644 index 00000000..2f93c426 --- /dev/null +++ b/docs/whalealData/use cases/ManuallyDeleteSourceData/index.html @@ -0,0 +1,5513 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + ManuallyDeleteSourceData - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

ManuallyDeleteSourceData

+ +

Manual Deletion of Source Data

+

When configuring table jobs in the "Table Job Configuration" section under the "Configuration Management" menu, there's an option for "Data Processing Method." By selecting "Manual Deletion," along with providing a batch value, you can control how the source data is managed after archiving.

+
    +
  • If you choose "Manual Deletion" and set a batch value, the source data won't be automatically deleted after archiving is completed. Instead, it will be marked as archived with the specified batch value.
    • +
+

To manually delete the source data after archiving:

+
    +
  1. Go to the "Task Monitoring (Cold)" section.
    2. +
  2. Find the task for which you want to delete the source data.
    3. +
  3. Click on the task to view its details.
    4. +
  4. Click "Modify Source Table Data Status" to change the status of the source table data.
    5. +
  5. Alternatively, click "Manual Deletion of Source Table Data" to manually delete the source table data.
    6. +
+

This provides you with control over when and how the source data is deleted after archiving, allowing you to manage your data archiving process according to your specific needs.

+

image-20230621152337512

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/use cases/RetryAbnormalTask/index.html b/docs/whalealData/use cases/RetryAbnormalTask/index.html new file mode 100644 index 00000000..a8a6ce06 --- /dev/null +++ b/docs/whalealData/use cases/RetryAbnormalTask/index.html @@ -0,0 +1,5510 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + RetryAbnormalTask - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

RetryAbnormalTask

+ +

Re-Execution of Failed Tasks

+

Under the "Task Management" menu, there are three sections for task monitoring: "Completed," "In Progress," and "Exception." For tasks that have encountered exceptions or failures, you have the option to manually trigger a re-execution of these tasks.

+

Here's how you can re-execute a failed task:

+
    +
  1. Go to the "Task Monitoring" section.
    2. +
  2. Click on the "Exception" tab to view tasks that encountered exceptions.
    3. +
  3. Find the specific task you want to re-execute and click on it to view its details.
    4. +
  4. Within the task details view, you'll find an option to "Re-Execute." Clicking this option will initiate the re-execution process.
    5. +
  5. The task will first be rolled back to its previous state, and then it will be re-executed.
    6. +
+

This functionality provides a way to address and resolve exceptions that may have occurred during task execution, allowing you to retry the task and ensure its successful completion.

+

image-20230621151925489

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/use cases/S3JobDemo/index.html b/docs/whalealData/use cases/S3JobDemo/index.html new file mode 100644 index 00000000..041a254a --- /dev/null +++ b/docs/whalealData/use cases/S3JobDemo/index.html @@ -0,0 +1,5763 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + S3JobDemo - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

S3JobDemo

+ +

S3 Job Demo

+

1. Adding Data Source and Target S3

+

Adding Data Source

+

Navigate to the "Configuration Management" menu and select "Data Source Management." Click the blue "Add" button and provide the necessary information for the data source. After testing and confirmation, save the data source. +image-20230621132912082

+

Adding Target S3

+

In the "Configuration Management" menu, select "Target Source Management." Click on the S3 section, then click the blue "Add" button to provide the relevant information for the target S3. After successful testing, confirm and save the target S3. +Add Target S3.png

+

2. Permission Assignment

+

After adding the data source and target S3, the administrator can assign database permissions to users through the "User Management" page under the "System Management" menu.

+

s3权限分配.png

+

3. Creating an S3 Job

+

Navigate to the "Configuration Management" menu and select "Table Job Configuration." On the S3 job configuration page, click the blue "Add" button to open the form. Choose the data source and target S3 you want to sync. The S3 job includes consistency verification, where you can set the required verification percentage. After synchronization, the platform will perform consistency checks on the synced data. You can also select the data handling method as either manual deletion or system deletion. After completion, the source table will be deleted according to batch numbers.

+

新建S3作业.png

+

4. Creating a Task

+

Adding a Task Configuration

+

Navigate to the "Task Configuration" menu and click the blue "Add" button to open the form. Fill in the required details based on your needs. If there are multiple jobs within the task, you can customize the execution mode, task timeout, and retry count. Select a notification strategy and add email addresses for notifications. After task completion, notifications will be sent based on the selected strategy.

+

新增s3任务配置.png

+

新增s3任务配置1.png

+

Adding Table Jobs to the Task

+

Click the "Add Job" button to select and add table jobs to the task. Once added, click "Confirm" to bind the jobs to the task. Multiple table jobs can be added to a single task.

+

添加s3表作业.png

+

Administrator Approval

+

After configuring the task, administrators can review and approve the task. Once approved, the task can be executed.

+

审核s3.png

+

5. Task Scheduling

+

Navigate to the "Task Scheduling" menu and check the status of the tasks. Click "Execute Now" to initiate immediate execution of a task.

+

s3任务调度.png

+

6. Task Monitoring

+

After executing or reaching the scheduled execution time, go to the "Task Monitoring" (S3) section under "Task Management" to view the status of the task.

+

s3任务监控.png

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/use cases/SMTPConfig/index.html b/docs/whalealData/use cases/SMTPConfig/index.html new file mode 100644 index 00000000..88aa5710 --- /dev/null +++ b/docs/whalealData/use cases/SMTPConfig/index.html @@ -0,0 +1,5514 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + SMTPConfig - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

SMTPConfig

+ +

SMTP Email Configuration

+

To configure SMTP email settings, follow these steps:

+
    +
  1. Click on the "System Management" menu and select "System Settings." This page is only accessible to the administrator user. The page displays SMTP configuration options.
    2. +
+

image-20230621105006375

+
    +
  1. +

    Fill in the required information for the SMTP configuration, including SMTP server, port, username, password, sender email, and recipient email.

    +
    2. +
  2. +

    After filling in the basic configuration, click the "Test Connection" button. If the test is successful and you receive a test email, the "Save" button will appear. Click on "Save" to save the SMTP configuration.

    +
    3. +
+

Once the SMTP configuration is saved, all task configuration alerts will be sent using the email settings provided in this configuration.

+

Please note that proper SMTP configuration is essential for receiving alerts and notifications from the system. Make sure to verify the accuracy of the configuration before saving it.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/use cases/SystemDeleteSourceData/index.html b/docs/whalealData/use cases/SystemDeleteSourceData/index.html new file mode 100644 index 00000000..15372311 --- /dev/null +++ b/docs/whalealData/use cases/SystemDeleteSourceData/index.html @@ -0,0 +1,5502 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + SystemDeleteSourceData - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

SystemDeleteSourceData

+ +

System Deletion of Source Data

+

When configuring table jobs in the "Table Job Configuration" section under the "Configuration Management" menu, you have the option to choose the data processing method. If you select "System Deletion," you will need to specify the batch value in the provided field. After the data archiving is completed, the system will automatically delete the source data based on the specified batch.

+

image-20230621152148702

+

Please exercise caution when using system deletion, as it will permanently remove source data based on the batch value you provide. Make sure to verify your configuration settings before proceeding.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/use cases/TaskExecutionMonitoring/index.html b/docs/whalealData/use cases/TaskExecutionMonitoring/index.html new file mode 100644 index 00000000..677e8951 --- /dev/null +++ b/docs/whalealData/use cases/TaskExecutionMonitoring/index.html @@ -0,0 +1,5502 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + TaskExecutionMonitoring - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

TaskExecutionMonitoring

+ +

Task Execution Monitoring

+

After clicking "Execute Now" on the task scheduling page or when the scheduled execution time is reached, different types of tasks will appear in different sections of the task monitoring page under the "Task Management" menu. For example, warm data tasks will appear in the "Task Monitoring (Warm)" section, cold data tasks will appear in the "Task Monitoring (Cold)" section, and so on. Each section provides information about the task's status, including "Completed," "In Progress," and "Exception."

+

image-20230621144555999

+

You can monitor the execution status of your tasks in the respective monitoring sections for warm, cold, and S3 data tasks.

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/use cases/UserLogin/index.html b/docs/whalealData/use cases/UserLogin/index.html new file mode 100644 index 00000000..a77df01c --- /dev/null +++ b/docs/whalealData/use cases/UserLogin/index.html @@ -0,0 +1,5501 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + UserLogin - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

UserLogin

+ +

User Login

+

To log in, enter the registered account credentials. For the first login of an administrator, the initial password is "123456." After logging in, you will be prompted to change the password. Other registered users have user-level permissions. Administrator users can assign permissions to new users.

+

image-20230621130728244

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/use cases/UserRegistration/index.html b/docs/whalealData/use cases/UserRegistration/index.html new file mode 100644 index 00000000..66d6e293 --- /dev/null +++ b/docs/whalealData/use cases/UserRegistration/index.html @@ -0,0 +1,5502 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + UserRegistration - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

UserRegistration

+ +

User Registration

+

After accessing the homepage, there is an "Register Now" button located at the bottom right corner of the login section. Clicking this button will take you to the registration page as shown in the second image. Fill in the required registration information and click "Register" to complete the process.

+

image-20230621115421937

+

image-20230621115335835

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file diff --git a/docs/whalealData/use cases/WarmJobDemo/index.html b/docs/whalealData/use cases/WarmJobDemo/index.html new file mode 100644 index 00000000..be332829 --- /dev/null +++ b/docs/whalealData/use cases/WarmJobDemo/index.html @@ -0,0 +1,5763 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + WarmJobDemo - Whaleal docs + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + Skip to content + + +
+
+ +
+ + + + +
+ + +
+ +
+ + + + + + + + + +
+
+ + + +
+
+
+ + + + + + + +
+
+
+ + + +
+
+
+ + + +
+
+
+ + + +
+
+ + + + + + + +

WarmJobDemo

+ +

Warm Job Demo

+

1. Add Data Source and Target Source

+

Add Data Source

+

Click on the "Data Source Management" page under the "Configuration Management" menu. On this page, click the blue "Add" button and fill in the relevant information for the data source. After passing the test, click "Confirm" to save this data source. +image-20230621132912082

+

Add Target Source

+

Click on the "Target Source Management" page under the "Configuration Management" menu. On this page, click the blue "Add" button and fill in the relevant information for the target source. After passing the test, click "Confirm" to save this target source. +image-20230621132755714

+

2. Permission Assignment

+

After adding the data source and target source, the administrator user can assign database permissions to users on the "User Management" page under the "System Management" menu.

+

image-20230621132523358

+

3. Create Warm Job

+

Click on the "Table Job Configuration" under the "Configuration Management" menu. On the warm data table job page, click the blue "Add" button to open the following form. Choose the data source and target source you want to synchronize. Table jobs have consistency verification functionality. After selecting "Yes," you can enter the desired verification percentage. After synchronization, the platform will perform consistency checks on the synchronized data. For MySQL-related jobs, you can choose the isolation level. When synchronizing from MongoDB to MongoDB, you can choose whether to synchronize Gridfs. If you choose "No," the platform will only synchronize ordinary documents. The data processing mode can be selected as manual deletion or system deletion. This feature will delete the source table by batch after synchronization is completed.

+

image-20230621134350471

+

4. Create Task

+

Add Task Configuration

+

Click on the "Task Configuration" under the "Task Configuration" menu. This page displays all tasks. Click the blue "Add" button to open the form shown in the second image. Fill in the task configuration according to your needs. If there are many table jobs in this task, you can set the execution mode, task timeout, and failure retry times according to your needs. After selecting a reminder strategy, you can add email addresses for receiving alert notifications. After the task is completed, alerts will be sent via email according to the selected strategy. +image-20230621142941634

+

image-20230621143119669

+

Add Table Jobs to the Task

+

Click on "Add Job" to open the data configuration shown in the third image. Check the desired job and click "OK" to bind this job to the newly created task. A task can be bound to multiple table jobs.

+

image-20230621143532776

+

Administrator Approval

+

After configuring the task, the administrator user can review the task configuration. After approval, the task can be executed.

+

image-20230619183730879

+

5. Task Scheduling

+

Click on "Task Scheduling" under the "Task Management" menu to view the status of the task. Click "Execute Now" to immediately run the task.

+

image-20230620133351561

+

6. Task Monitoring

+

After clicking "Execute Now" on the task scheduling page or when the task execution time arrives, you can view the execution status of the task in the task monitoring (Warm) section under the "Task Management" menu.

+

image-20230620142632337

+ + + + + + + + + + + + + +
+
+ + + +
+ +
+ + + +
+
+
+
+ +
+ + + + + + + + + + \ No newline at end of file