[AMORO-2369]Optimize project module structure (apache#2374)

* change module structure * add ams/dist * rollback docs format change * Rollback README format change * Rollback README format change * Rollback docs format change * Change Github CI configuration & docker build scripts * Change Github CI configurations
ShawHee · Dec 29, 2023 · c095fa5 · c095fa5
1 parent dd50527
commit c095fa5
Show file tree

Hide file tree

Showing 831 changed files with 240 additions and 201 deletions.
diff --git a/.github/workflows/core-hadoop2-ci.yml b/.github/workflows/core-hadoop2-ci.yml
@@ -20,9 +20,9 @@ on:
     paths:
       - "ams/**"
       - "core/**"
-      - "flink/**"
-      - "hive/**"
-      - "spark/**"
+      - "mixed/flink/**"
+      - "mixed/hive/**"
+      - "mixed/spark/**"
       - "pom.xml"
 
 jobs:
@@ -42,7 +42,7 @@ jobs:
         run: mvn validate
 
       - name: Build all module with Maven
-        run: mvn clean install -pl '!trino' -Djacoco.flink.skip=true -B -ntp -Dhadoop=v2
+        run: mvn clean install -pl '!mixed/trino' -Djacoco.flink.skip=true -B -ntp -Dhadoop=v2
 
       - name: Code coverage
         uses: codecov/codecov-action@v3

diff --git a/.github/workflows/core-hadoop3-ci.yml b/.github/workflows/core-hadoop3-ci.yml
@@ -20,9 +20,9 @@ on:
     paths:
       - "ams/**"
       - "core/**"
-      - "flink/**"
-      - "hive/**"
-      - "spark/**"
+      - "mixed/flink/**"
+      - "mixed/hive/**"
+      - "mixed/spark/**"
       - "pom.xml"
 
 jobs:
@@ -42,7 +42,7 @@ jobs:
         run: mvn validate
 
       - name: Build all module with Maven
-        run: mvn clean install -pl '!trino' -Djacoco.flink.skip=true -B -ntp
+        run: mvn clean install -pl '!mixed/trino' -Djacoco.flink.skip=true -B -ntp
 
       - name: Code coverage
         uses: codecov/codecov-action@v3

diff --git a/.github/workflows/docker-images.yml b/.github/workflows/docker-images.yml
@@ -40,7 +40,7 @@ jobs:
     if: ${{ startsWith(github.repository, 'NetEase/') }}
     strategy:
       matrix:
-        hadoop: ["v3"]
+        hadoop: [ "v3" ]
     steps:
       - uses: actions/checkout@v3
       - name: Set up JDK 8
@@ -85,7 +85,7 @@ jobs:
           echo "MVN_HADOOP=-Dhadoop=${{ matrix.hadoop }}" >> $GITHUB_ENV
 
       - name: Build dist module with Maven
-        run: mvn clean install -pl 'dist' -am -e ${MVN_HADOOP} -DskipTests -B -ntp
+        run: mvn clean install -pl 'ams/dist' -am -e ${MVN_HADOOP} -DskipTests -B -ntp
 
       - name: Build and Push Amoro Docker Image
         uses: docker/build-push-action@v4
@@ -218,7 +218,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Build optimizer module with Maven
-        run: mvn clean package -pl 'flink/v1.17/flink-runtime' -am -e -DskipTests -B -ntp
+        run: mvn clean package -pl 'mixed/flink/v1.17/flink-runtime' -am -e -DskipTests -B -ntp
 
       - name: Build and Push Flink Optimizer Docker Image
         uses: docker/build-push-action@v4

diff --git a/.github/workflows/trino-ci.yml b/.github/workflows/trino-ci.yml
@@ -20,8 +20,8 @@ on:
     paths:
       - "ams/**"
       - "core/**"
-      - "hive/**"
-      - "trino/**"
+      - "mixed/hive/**"
+      - "mixed/trino/**"
       - "pom.xml"
 
 jobs:
@@ -40,10 +40,10 @@ jobs:
         run: mvn validate -P trino-spotless
 
       - name: Install dependency with Maven
-        run: mvn clean install -DskipTests -pl 'ams/api,core,hive' -Dhadoop=v2 -am -B -P trino-spotless
+        run: mvn clean install -DskipTests -pl 'ams/api,core,mixed/hive' -Dhadoop=v2 -am -B -P trino-spotless
 
       - name: Build with Maven
-        run: mvn clean test -pl 'trino' -B -ntp -Dhadoop=v2 -P trino-spotless
+        run: mvn clean test -pl 'mixed/trino' -B -ntp -Dhadoop=v2 -P trino-spotless
 
       - name: Code coverage
         uses: codecov/codecov-action@v3

diff --git a/.gitignore b/.gitignore
@@ -28,8 +28,6 @@ venv/
 *.egg-info/
 test-reports
 build/
-dist/
-sdist/
 .coverage
 coverage.xml
 .pytest_cache/
@@ -62,8 +60,12 @@ target/
 # local config
 conf/
 
+
 # test resources
 *unit_test_base_tmp/
 
 */.gitignore
-*.tgz
+*.tgz
+
+!ams/dist/src/main/arctic-bin/bin/
+!ams/dist/src/main/arctic-bin/conf/
diff --git a/README.md b/README.md
@@ -55,7 +55,7 @@ Here is the architecture diagram of Amoro:
     * Terminal: SQL command-line tools, provide various implementations like local Spark and Kyuubi.
     * LogStore: Provide millisecond to second level SLAs for real-time data processing based on message queues like Kafka and Pulsar.
 
-## Supported table formats 
+## Supported table formats
 
 Amoro can manage tables of different table formats, similar to how MySQL/ClickHouse can choose different storage engines.
 Amoro meets diverse user needs by using different table formats. Currently, Amoro supports four table formats:
@@ -102,15 +102,16 @@ Amoro contains modules as below:
     - `ams-dashboard` is the dashboard frontend for ams
     - `ams-server` is the backend server for ams
     - `ams-optimizer` provides default optimizer implementation
-- `amoro-hive` integrates with Apache Hive and implements Mixed Hive format
-- `amoro-flink` provides Flink connectors for Mixed format tables (use amoro-flink-runtime for a shaded version)
-- `amoro-spark` provides Spark connectors for Mixed format tables (use amoro-spark-runtime for a shaded version)
-- `amoro-trino` provides Trino connectors for Mixed format tables
+- `amoro-mixed` provides Mixed format implementation
+    - `amoro-hive` integrates with Apache Hive and implements Mixed Hive format
+    - `amoro-flink` provides Flink connectors for Mixed format tables (use amoro-flink-runtime for a shaded version)
+    - `amoro-spark` provides Spark connectors for Mixed format tables (use amoro-spark-runtime for a shaded version)
+    - `amoro-trino` provides Trino connectors for Mixed format tables
 
 
 ## Building
 
-Amoro is built using Maven with Java 1.8 and Java 17(only for `trino` module).
+Amoro is built using Maven with Java 1.8 and Java 17(only for `mixed/trino` module).
 
 * To build Trino module need config `toolchains.xml` in `${user.home}/.m2/` dir, the content is
 
@@ -132,15 +133,15 @@ Amoro is built using Maven with Java 1.8 and Java 17(only for `trino` module).
 
 * To invoke a build and run tests: `mvn package -P toolchain`
 * To skip tests: `mvn -DskipTests package -P toolchain`
-* To package without trino module and JAVA 17 dependency: `mvn clean package -DskipTests -pl '!trino'`
+* To package without trino module and JAVA 17 dependency: `mvn clean package -DskipTests -pl '!mixed/trino'`
 * To build with hadoop 2.x(the default is 3.x) `mvn clean package -DskipTests -Dhadoop=v2`
 * To indicate flink version for optimizer(the default is 1.14, 1.15 and 1.16 are available)
   `mvn clean package -DskipTests -Doptimizer.flink=1.15`
 
 >Spotless is skipped by default in `trino` module. So if you want to perform checkstyle when building `trino` module, you must be in a Java 17 environment.
 
-* To invoke a build include `trino` module in Java 17 environment: `mvn clean package -DskipTests -P trino-spotless`
-* To only build `trino` and its dependent modules in Java 17 environment: `mvn clean package -DskipTests -P trino-spotless -pl 'trino' -am`
+* To invoke a build include `mixed/trino` module in Java 17 environment: `mvn clean package -DskipTests -P trino-spotless`
+* To only build `mixed/trino` and its dependent modules in Java 17 environment: `mvn clean package -DskipTests -P trino-spotless -pl 'trino' -am`
 ## Quickstart
 
 Visit [https://amoro.netease.com/quick-demo/](https://amoro.netease.com/quick-demo/) to quickly

diff --git a/dist/pom.xml → ams/dist/pom.xml b/dist/pom.xml → ams/dist/pom.xml
@@ -22,12 +22,12 @@
     <modelVersion>4.0.0</modelVersion>
     <parent>
         <groupId>com.netease.amoro</groupId>
-        <artifactId>amoro-parent</artifactId>
+        <artifactId>amoro-ams</artifactId>
         <version>0.7.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>dist</artifactId>
-    <name>Amoro Project Dist</name>
+    <name>Amoro Project AMS Dist</name>
     <url>https://amoro.netease.com</url>
 
     <properties>

diff --git a/dist/src/main/arctic-bin/bin/ams.sh → ams/dist/src/main/arctic-bin/bin/ams.sh b/dist/src/main/arctic-bin/bin/ams.sh → ams/dist/src/main/arctic-bin/bin/ams.sh
diff --git a/dist/src/main/arctic-bin/bin/load-config.sh → ...st/src/main/arctic-bin/bin/load-config.sh b/dist/src/main/arctic-bin/bin/load-config.sh → ...st/src/main/arctic-bin/bin/load-config.sh
diff --git a/dist/src/main/arctic-bin/bin/optimizer.sh → ...dist/src/main/arctic-bin/bin/optimizer.sh b/dist/src/main/arctic-bin/bin/optimizer.sh → ...dist/src/main/arctic-bin/bin/optimizer.sh
diff --git a/dist/src/main/arctic-bin/conf/config.yaml → ...dist/src/main/arctic-bin/conf/config.yaml b/dist/src/main/arctic-bin/conf/config.yaml → ...dist/src/main/arctic-bin/conf/config.yaml
diff --git a/dist/src/main/arctic-bin/conf/jvm.properties → ...t/src/main/arctic-bin/conf/jvm.properties b/dist/src/main/arctic-bin/conf/jvm.properties → ...t/src/main/arctic-bin/conf/jvm.properties
diff --git a/dist/src/main/arctic-bin/conf/log4j2.xml → ams/dist/src/main/arctic-bin/conf/log4j2.xml b/dist/src/main/arctic-bin/conf/log4j2.xml → ams/dist/src/main/arctic-bin/conf/log4j2.xml
diff --git a/.../main/arctic-bin/conf/optimize/log4j2.xml → .../main/arctic-bin/conf/optimize/log4j2.xml b/.../main/arctic-bin/conf/optimize/log4j2.xml → .../main/arctic-bin/conf/optimize/log4j2.xml
diff --git a/.../main/arctic-bin/metrics/log_emitter.yaml → .../main/arctic-bin/metrics/log_emitter.yaml b/.../main/arctic-bin/metrics/log_emitter.yaml → .../main/arctic-bin/metrics/log_emitter.yaml
diff --git a/dist/src/main/assemblies/bin.xml → ams/dist/src/main/assemblies/bin.xml b/dist/src/main/assemblies/bin.xml → ams/dist/src/main/assemblies/bin.xml
@@ -29,13 +29,15 @@
 
     <files>
         <file>
-            <source>../ams/optimizer/flink-optimizer/target/flink-optimizer-${project.version}-jar-with-dependencies.jar</source>
+            <source>
+                ../optimizer/flink-optimizer/target/flink-optimizer-${project.version}-jar-with-dependencies.jar
+            </source>
             <outputDirectory>plugin/optimizer/flink</outputDirectory>
             <destName>optimizer-job.jar</destName>
             <fileMode>0644</fileMode>
         </file>
         <file>
-            <source>../ams/server/target/amoro-ams-server-${project.version}.jar</source>
+            <source>../server/target/amoro-ams-server-${project.version}.jar</source>
             <outputDirectory>lib/</outputDirectory>
             <destName>amoro-ams-server-${project.version}.jar</destName>
             <fileMode>0644</fileMode>
@@ -59,22 +61,22 @@
             <fileMode>0644</fileMode>
         </fileSet>
         <fileSet>
-            <directory>../ams/server/src/main/resources/mysql</directory>
+            <directory>../server/src/main/resources/mysql</directory>
             <outputDirectory>conf/mysql</outputDirectory>
             <fileMode>0644</fileMode>
         </fileSet>
         <fileSet>
-            <directory>../ams/server/src/main/resources/postgres</directory>
+            <directory>../server/src/main/resources/postgres</directory>
             <outputDirectory>conf/postgres</outputDirectory>
             <fileMode>0644</fileMode>
         </fileSet>
         <fileSet>
-            <directory>../ams/server/src/main/resources/derby</directory>
+            <directory>../server/src/main/resources/derby</directory>
             <outputDirectory>conf/derby</outputDirectory>
             <fileMode>0644</fileMode>
         </fileSet>
         <fileSet>
-            <directory>../ams/server/target/amoro-ams-server-dependency/lib</directory>
+            <directory>../server/target/amoro-ams-server-dependency/lib</directory>
             <outputDirectory>lib/</outputDirectory>
             <fileMode>0644</fileMode>
         </fileSet>

diff --git a/ams/optimizer/common/pom.xml b/ams/optimizer/common/pom.xml
@@ -29,7 +29,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>optimizer-common</artifactId>
-    <name>Amoro Project Optimizer Common</name>
+    <name>Amoro Project AMS Optimizer Common</name>
     <url>https://amoro.netease.com</url>
 
     <dependencies>
@@ -40,7 +40,7 @@
 
         <dependency>
             <groupId>com.netease.amoro</groupId>
-            <artifactId>amoro-hive</artifactId>
+            <artifactId>amoro-mixed-hive</artifactId>
         </dependency>
 
         <dependency>

diff --git a/ams/optimizer/flink-optimizer/pom.xml b/ams/optimizer/flink-optimizer/pom.xml
@@ -29,7 +29,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>flink-optimizer</artifactId>
-    <name>Amoro Project Flink Optimizer</name>
+    <name>Amoro Project AMS Flink Optimizer</name>
     <url>https://amoro.netease.com</url>
 
     <properties>

diff --git a/ams/optimizer/pom.xml b/ams/optimizer/pom.xml
@@ -29,7 +29,7 @@
 
     <artifactId>amoro-optimizer</artifactId>
     <packaging>pom</packaging>
-    <name>Amoro Project Optimizer Parent</name>
+    <name>Amoro Project AMS Optimizer Parent</name>
     <url>https://amoro.netease.com</url>
 
     <modules>

diff --git a/ams/optimizer/standalone-optimizer/pom.xml b/ams/optimizer/standalone-optimizer/pom.xml
@@ -29,7 +29,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>standalone-optimizer</artifactId>
-    <name>Amoro Project Standalone Optimizer</name>
+    <name>Amoro Project AMS Standalone Optimizer</name>
     <url>https://amoro.netease.com</url>
 
 

diff --git a/ams/pom.xml b/ams/pom.xml
@@ -34,9 +34,10 @@
 
     <modules>
         <module>api</module>
-        <module>optimizer</module>
         <module>server</module>
         <module>dashboard</module>
+        <module>optimizer</module>
+        <module>dist</module>
     </modules>
 
 </project>
diff --git a/ams/server/pom.xml b/ams/server/pom.xml
@@ -43,7 +43,7 @@
 
         <dependency>
             <groupId>com.netease.amoro</groupId>
-            <artifactId>amoro-hive</artifactId>
+            <artifactId>amoro-mixed-hive</artifactId>
             <exclusions>
                 <exclusion>
                     <groupId>org.eclipse.jetty.orbit</groupId>
@@ -132,7 +132,7 @@
 
         <dependency>
             <groupId>com.netease.amoro</groupId>
-            <artifactId>amoro-spark-${terminal.spark.major.version}</artifactId>
+            <artifactId>amoro-mixed-spark-${terminal.spark.major.version}</artifactId>
             <version>${project.version}</version>
             <scope>runtime</scope>
         </dependency>
@@ -340,7 +340,7 @@
 
         <dependency>
             <groupId>com.netease.amoro</groupId>
-            <artifactId>amoro-hive</artifactId>
+            <artifactId>amoro-mixed-hive</artifactId>
             <version>${project.version}</version>
             <scope>test</scope>
             <type>test-jar</type>

diff --git a/docker/amoro/Dockerfile b/docker/amoro/Dockerfile
@@ -35,7 +35,7 @@ RUN apt-get update \
     && apt-get install -y unzip
 
 RUN AMORO_VERSION=`cat pom.xml | grep 'amoro-parent' -C 3 | grep -Eo '<version>.*</version>' | awk -F'[><]' '{print $3}'` \
-    && cp dist/target/*.zip /usr/local \
+    && cp ams/dist/target/*.zip /usr/local \
     && unzip /usr/local/amoro-${AMORO_VERSION}-bin.zip -d /usr/local \
     && rm /usr/local/amoro-${AMORO_VERSION}/plugin/optimizer/flink -rf \
     && mv /usr/local/amoro-${AMORO_VERSION} /usr/local/amoro \

diff --git a/docker/build.sh b/docker/build.sh
@@ -215,10 +215,10 @@ function build_amoro() {
   local IMAGE_TAG=$AMORO_TAG
   print_image $IMAGE_REF $IMAGE_TAG
 
-  local DIST_FILE=${PROJECT_HOME}/dist/target/amoro-${AMORO_VERSION}-bin.zip
+  local DIST_FILE=${PROJECT_HOME}/ams/dist/target/amoro-${AMORO_VERSION}-bin.zip
 
   if [ ! -f "${DIST_FILE}" ]; then
-    local BUILD_CMD="mvn clean package -am -e -pl dist -DskipTests "
+    local BUILD_CMD="mvn clean package -am -e -pl ams/dist -DskipTests "
     echo "Amoro dist package is not exists in ${DIST_FILE}"
     echo "please check file or run '$BUILD_CMD' first"
   fi
@@ -234,18 +234,18 @@ function build_quickdemo() {
     local IMAGE_REF=arctic163/quickdemo
     local IMAGE_TAG=$AMORO_TAG
 
-    local FLINK_CONNECTOR_BINARY=${PROJECT_HOME}/flink/v${FLINK_MAJOR_VERSION}/flink-runtime/target/amoro-flink-runtime-${FLINK_MAJOR_VERSION}-${AMORO_VERSION}.jar
+    local FLINK_CONNECTOR_BINARY=${PROJECT_HOME}/mixed/flink/v${FLINK_MAJOR_VERSION}/flink-runtime/target/amoro-mixed-flink-runtime-${FLINK_MAJOR_VERSION}-${AMORO_VERSION}.jar
 
     if [ ! -f "${FLINK_CONNECTOR_BINARY}" ]; then
-        echo "amoro-flink-connector not exists in ${FLINK_CONNECTOR_BINARY}, run 'mvn clean package -pl !trino' first. "
+        echo "amoro-mixed-flink-connector not exists in ${FLINK_CONNECTOR_BINARY}, run 'mvn clean package -pl !mixed/trino' first. "
         exit  1
     fi
 
     if [ "${ALSO_MAKE}" == "true" ]; then
         echo "Build dependency Amoro image."
         build_amoro
         if [ "$?" -ne 0 ]; then
-          echo "Build required Amor image failed."
+          echo "Build required Amoro image failed."
           exit 1
         fi
     fi

diff --git a/docker/quickdemo/Dockerfile b/docker/quickdemo/Dockerfile
@@ -34,8 +34,8 @@ WORKDIR /workspace/amoro
 RUN AMORO_VERSION=`cat pom.xml | grep 'amoro-parent' -C 3 | grep -Eo '<version>.*</version>' | awk -F'[><]' '{print $3}'` \
     && FLINK_VERSION=${FLINK_VERSION} \
     && FLINK_MAJOR_VERSION="${FLINK_VERSION%.*}" \
-    && cp flink/v${FLINK_MAJOR_VERSION}/flink-runtime/target/amoro-flink-runtime-${FLINK_MAJOR_VERSION}-${AMORO_VERSION}.jar \
-      /opt/amoro-flink-runtime-${FLINK_MAJOR_VERSION}-${AMORO_VERSION}.jar \
+    && cp mixed/flink/v${FLINK_MAJOR_VERSION}/flink-runtime/target/amoro-mixed-flink-runtime-${FLINK_MAJOR_VERSION}-${AMORO_VERSION}.jar \
+      /opt/amoro-mixed-flink-runtime-${FLINK_MAJOR_VERSION}-${AMORO_VERSION}.jar \
     && rm -rf /workspace/amoro