generate jave files using maven

build.sh

@@ -41,8 +41,7 @@ if [[ ! -f ${DORIS_THIRDPARTY}/installed/lib/libs2.a ]]; then
     ${DORIS_THIRDPARTY}/build-thirdparty.sh
 fi
 
-#PARALLEL=$[$(nproc)/4+1]
-PARALLEL=12
+PARALLEL=$[$(nproc)/4+1]
 
 # Check args
 usage() {
@@ -148,6 +147,7 @@ echo "Build generated code"
 cd ${DORIS_HOME}/gensrc
 if [ ${CLEAN} -eq 1 ]; then
    make clean
+   rm -rf ${DORIS_HOME}/fe/fe-core/target
 fi
 # DO NOT using parallel make(-j) for gensrc
 make

docs/en/developer-guide/fe-eclipse-dev.md

@@ -34,23 +34,23 @@ under the License.
 
 ### Code Generation
 
-The FE module requires part of the generated code, such as Thrift, Protobuf and other frameworks. This part needs to be generated by compiling the contents of the `gensrc/` directory in the Linux environment.
+The FE module requires part of the generated code, such as Thrift, Protobuf, Jflex, CUP and other frameworks. 
 
-1. Under Linux, enter the source code directory and execute the following command:
+1. Under Linux, enter the source code directory `fe` and execute the following command:
 
-    ```
-    sh build.sh --clean --fe
-    ```
+   ```
+    mvn  generate-sources
+   ```
 
-2. Then package the generated `gensrc/build/java` directory:
+2. If use windows as development environment, then package the generated `fe/fe-core/target/generated-sources` directory:
 
-    `cd gensrc/build/ && tar czf java.tar.gz java/`
+   `fe/fe-core/target/ && tar czf java.tar.gz generated-sources/`
 
-3. Copy `java.tar.gz` to the `fe/` directory of the development environment and unzip
+3. Copy `java.tar.gz` to the `fe/fe-core/target/` directory of the development environment and unzip
 
     ```
-    cp java.tar.gz /path/to/doris/fe/
-    cd /path/to/doris/fe/ && tar xzf java.tar.gz
+    cp java.tar.gz /path/to/doris/fe/fe-core/target/
+    cd /path/to/doris/fe/fe-core/target/ && tar xzf java.tar.gz
     ```
 
 ## Import FE project
@@ -126,13 +126,12 @@ You can directly start an FE process in Ecplise to facilitate debugging the code
 
 ## Code Update
 
-1. Update lexical and grammar files
+1. Update lexical and grammar files or proto and thrift files
 
-    If you modified `fe/src/main/cup/sql_parser.cup` or `fe/src/main/jflex/sql_scanner.flex` file. You need to execute the following commands in the `fe/` directory:
+    If you modified `fe/src/main/cup/sql_parser.cup` or `fe/src/main/jflex/sql_scanner.flex` file or proto and thrift files. You need to execute the following commands in the `fe/` directory:
 
     ```
-    mvn de.jflex:maven-jflex-plugin:1.4.3:generate
-    mvn net.sourceforge.czt.dev:cup-maven-plugin:1.6-cdh:generate
+    mvn  generate-sources
     ```
 
     Then refresh the project in Eclipse.

docs/zh-CN/developer-guide/fe-eclipse-dev.md

@@ -34,23 +34,22 @@ under the License.
 
 ### 代码生成
 
-FE 模块需要部分生成代码，如 Thrift、Protobuf 等框架的生成代码。这部分需要在 Linux 环境中通过编译 `gensrc/` 目录中的内容生成。
+FE 模块需要部分生成代码，如 Thrift、Protobuf, jflex, cup 等框架的生成代码。这部分需要在 Linux 或者 Mac环境生成。
 
-1. 在 Linux 下，进入到源码目录，执行下面的命令：
-
-    ```
-    sh build.sh --clean --fe
-    ```
+1. 在 Linux 下， 进入 `fe 目录下执行以下命令：
+   ```
+   mvn  generate-sources
+   ```
 
-2. 然后将生成的 `gensrc/build/java` 目录打包：
+2. 如果使用window开发 需要将生成的 `fe/fe-core/target/generated-sources` 目录打包：
 
-    `cd gensrc/build/ && tar czf java.tar.gz java/`
+    `fe/fe-core/target/ && tar czf java.tar.gz generated-sources/`
 
-3. 将 `java.tar.gz` 拷贝到开发环境的 `fe/` 目录下，并解压
+3. 将 `java.tar.gz` 拷贝到开发环境的 `fe/fe-core/target/` 目录下，并解压
 
     ```
-    cp java.tar.gz /path/to/doris/fe/
-    cd /path/to/doris/fe/ && tar xzf java.tar.gz
+    cp java.tar.gz /path/to/doris/fe/fe-core/target/
+    cd /path/to/doris/fe/fe-core/target/ && tar xzf java.tar.gz
     ```
 
 ## 导入 FE 工程
@@ -126,13 +125,12 @@ java.lang.Exception: Method xxxx should have no parameters
 
 ## 代码更新
 
-1. 更新词法、语法文件
+1. 更新词法、语法文件或者thrift 和proto 文件
 
-    如果修改了 `fe/src/main/cup/sql_parser.cup` 或者 `fe/src/main/jflex/sql_scanner.flex` 文件。则需在 `fe/` 目录下执行以下命令：
+    如果修改了 `fe/fe-core/src/main/cup/sql_parser.cup` 或者 `fe/fe-core/src/main/jflex/sql_scanner.flex`文件或者proto 和thrift 文件。则需在 `fe` 目录下执行以下命令：
 
     ```
-    mvn de.jflex:maven-jflex-plugin:1.4.3:generate
-    mvn net.sourceforge.czt.dev:cup-maven-plugin:1.6-cdh:generate
+    mvn  generate-sources
     ```
 
     之后在 Eclipse 中刷新工程即可。

docs/zh-CN/developer-guide/fe-idea-dev.md

@@ -30,27 +30,17 @@ under the License.
 
 JDK1.8+  , Intellj IDEA
 
-1.linux上编译好fe前端代码，主要目的是获取自动生成的代码，加入到前段工程里面去用于在idea中编译fe工程
-
-在linux下，进入到源码目录，执行下面的命令：
-
+1. 从 https://github.com/apache/incubator-doris.git 下载源码到本地
+2. 使用Intellj IDEA 打开代码根目录
+3. 如果仅进行fe开发而没有编译过thirdparty，则需要安装thrift，并将thrift 复制或者连接到 `thirdparty/installed/bin` 目录下
+4. 如果是Mac 或者 Linux 环境 可以通过 如下命令生成自动生成代码：
 ```
-$ sh build.sh --clean --fe
+cd fe
+mvn  generate-sources
 ```
-
-然后将 gensrc目录打包，拿出来，如下图
-
-![](/images/DEBUG1.png)
-
-2.在windows下解压gensrc.tar.gz,解压后的目录如下图：
-
-![](/images/DEBUG2.png)
-
-3.进入build/java,将红色框出的部分整个拷贝到源码的fe/src/main/java目录下
-
-
-
-![](/images/DEBUG3.png)
+或者通过图形界面运行运行maven 命令生成
+![](/images/gen_code.png)
+如果使用windows环境可能会有make命令和sh脚本无法执行的情况 可以通过拷贝linux上的 `fe/fe-core/target/generated-sources` 目录拷贝到相应的目录的方式实现，也可以通过docker 镜像挂载本地目录之后，在docker 内部生成自动生成代码，可以参照编译一节
 
 ## 2.调试
 

fe/fe-core/pom.xml

@@ -20,7 +20,7 @@ under the License.
 -->
 
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
 
     <parent>
@@ -37,8 +37,23 @@ under the License.
     <properties>
         <doris.home>${basedir}/../../</doris.home>
         <fe_ut_parallel>${env.FE_UT_PARALLEL}</fe_ut_parallel>
+        <doris.thridparty>${basedir}/../../thirdparty</doris.thridparty>
     </properties>
 
+    <profiles>
+        <profile>
+            <id>thirdparty</id>
+            <activation>
+                <property>
+                    <name>env.DORIS_THIRDPARTY</name>
+                </property>
+            </activation>
+            <properties>
+                <doris.thridparty>${env.DORIS_THIRDPARTY}</doris.thridparty>
+            </properties>
+        </profile>
+    </profiles>
+
     <dependencies>
         <!-- https://mvnrepository.com/artifact/cglib/cglib -->
         <dependency>
@@ -428,7 +443,7 @@ under the License.
             <artifactId>spark-launcher_2.12</artifactId>
         </dependency>
 
-		<!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql_2.12 -->
+        <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-sql_2.12 -->
         <dependency>
             <groupId>org.apache.spark</groupId>
             <artifactId>spark-sql_2.12</artifactId>
@@ -440,6 +455,25 @@ under the License.
         <finalName>palo-fe</finalName>
 
         <plugins>
+            <!--thrift-->
+            <plugin>
+                <groupId>org.apache.thrift.tools</groupId>
+                <artifactId>maven-thrift-plugin</artifactId>
+                <version>0.1.11</version>
+                <configuration>
+                    <thriftExecutable>${doris.thridparty}/installed/bin/thrift</thriftExecutable>
+                    <thriftSourceRoot>${doris.home}/gensrc/thrift</thriftSourceRoot>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>thrift-sources</id>
+                        <phase>generate-sources</phase>
+                        <goals>
+                            <goal>compile</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
             <!--jcup-->
             <plugin>
                 <groupId>net.sourceforge.czt.dev</groupId>
@@ -506,8 +540,23 @@ under the License.
             <plugin>
                 <groupId>org.codehaus.mojo</groupId>
                 <artifactId>exec-maven-plugin</artifactId>
-                <version>1.6.0</version>
+                <version>3.0.0</version>
                 <executions>
+                    <execution>
+                        <id>make-dir</id>
+                        <phase>generate-sources</phase>
+                        <goals>
+                            <goal>exec</goal>
+                        </goals>
+                        <configuration>
+                            <executable>mkdir</executable>
+                            <arguments>
+                                <argument>-p</argument>
+                                <argument>${basedir}/target/generated-sources/proto</argument>
+                            </arguments>
+                            <skip>${skip.plugin}</skip>
+                        </configuration>
+                    </execution>
                     <execution>
                         <id>gensrc</id>
                         <phase>generate-sources</phase>
@@ -518,7 +567,7 @@ under the License.
                             <executable>make</executable>
                             <arguments>
                                 <argument>-C</argument>
-                                <argument>${doris.home}/gensrc/</argument>
+                                <argument>${doris.home}/gensrc/script</argument>
                             </arguments>
                             <skip>${skip.plugin}</skip>
                         </configuration>
@@ -535,7 +584,7 @@ under the License.
                             <arguments>
                                 <argument>-jar</argument>
                                 <argument>${settings.localRepository}/com/baidu/jprotobuf/${jprotobuf.version}/jprotobuf-${jprotobuf.version}-jar-with-dependencies.jar</argument>
-                                <argument>--java_out=${doris.home}/gensrc/build/java/</argument>
+                                <argument>--java_out=${basedir}/target/generated-sources/proto</argument>
                                 <argument>${doris.home}/gensrc/proto/internal_service.proto</argument>
                             </arguments>
                             <skip>${skip.plugin}</skip>
@@ -559,7 +608,8 @@ under the License.
                         <configuration>
                             <sources>
                                 <!-- add arbitrary num of src dirs here -->
-                                <source>${doris.home}/gensrc/build/java/</source>
+                                <source>${basedir}/target/generated-sources/build/</source>
+                                <source>${basedir}/target/generated-sources/proto/</source>
                             </sources>
                         </configuration>
                     </execution>

gensrc/script/Makefile

@@ -18,6 +18,7 @@
 # generate files with python
 
 BUILD_DIR = ${CURDIR}/../build/
+FE_TARGET_DIR = ${CURDIR}/../../fe/fe-core/target/generated-sources/build
 
 # Prerequisites on the right side of '|' is only order
 all: gen_builtins gen_version
@@ -45,7 +46,7 @@ gen_vec_func: ${GEN_VEC_FUNC_OUTPUT}
 .PHONY: gen_vec_func
 
 # generate 
-GEN_BUILTINS_OUTPUT = ${BUILD_DIR}/java/org/apache/doris/builtins/ScalarBuiltins.java
+GEN_BUILTINS_OUTPUT = ${FE_TARGET_DIR}/org/apache/doris/builtins/ScalarBuiltins.java
 
 ${GEN_BUILTINS_OUTPUT}: doris_builtins_functions.py gen_builtins_functions.py ${GEN_FUNC_OUTPUT} ${GEN_VEC_FUNC_OUTPUT}
 	cd ${BUILD_DIR}/python && ${PYTHON} ${CURDIR}/gen_builtins_functions.py

gensrc/script/gen_build_version.sh

@@ -45,7 +45,7 @@ then
 fi
 
 if [[ -z ${DORIS_TEST_BINARY_DIR} ]]; then
-    if [ -e ${DORIS_HOME}/gensrc/build/java/org/apache/doris/common/Version.java \
+    if [ -e ${DORIS_HOME}/fe/fe-core/target/generated-sources/build/org/apache/doris/common/Version.java \
          -a -e ${DORIS_HOME}/gensrc/build/gen_cpp/version.h ]; then
         exit
     fi
@@ -91,7 +91,7 @@ java_version_str=$(echo $java_version_str | sed -e 's/"/\\"/g')
 echo "get java cmd: $java_cmd"
 echo "get java version: $java_version_str"
 
-VERSION_PACKAGE="${DORIS_HOME}/gensrc/build/java/org/apache/doris/common"
+VERSION_PACKAGE="${DORIS_HOME}/fe/fe-core/target/generated-sources/build/org/apache/doris/common"
 mkdir -p ${VERSION_PACKAGE}
 cat >"${VERSION_PACKAGE}/Version.java" <<EOF
 // Licensed to the Apache Software Foundation (ASF) under one

gensrc/script/gen_builtins_functions.py

@@ -47,7 +47,8 @@
   }\n\
 }\n'
 
-FE_PATH = "../java/org/apache/doris/builtins/"
+FE_PATH = "../../../fe/fe-core/target/generated-sources/build/org/apache/doris/builtins/"
+print FE_PATH
 
 # This contains all the metadata to describe all the builtins.
 # Each meta data entry is itself a map to store all the meta data

gensrc/thrift/Makefile

@@ -22,18 +22,14 @@ THRIFT = ${DORIS_THIRDPARTY}/installed/bin/thrift
 
 SOURCES = $(shell find ${CURDIR} -name "*.thrift")
 OBJECTS = $(patsubst ${CURDIR}/%.thrift, ${BUILD_DIR}/gen_cpp/%_types.cpp, ${SOURCES})
-# It doesn't exist really, just use this to compile all thrift to generate java file
-JAVA_OBJECTS = $(patsubst ${CURDIR}/%.thrift, ${BUILD_DIR}/java/.%.tm, ${SOURCES})
 
 GEN_SOURCES = $(shell find ${THRIFT} -name "*.thrift")
 GEN_OBJECTS = $(patsubst ${BUILD_DIR}/thrift/%.thrift, ${BUILD_DIR}/gen_cpp/%_types.cpp, ${GEN_SOURCES})
-GEN_JAVA_OBJECTS = $(patsubst ${BUILD_DIR}/thrift/%.thrift, ${BUILD_DIR}/java/.%.tm, ${GEN_SOURCES})
 
-all: ${GEN_OBJECTS} ${OBJECTS} ${GEN_JAVA_OBJECTS} ${JAVA_OBJECTS}
+all: ${GEN_OBJECTS} ${OBJECTS}
 .PHONY: all
 
 THRIFT_CPP_ARGS = -I ${CURDIR} -I ${BUILD_DIR}/thrift/ --gen cpp -out ${BUILD_DIR}/gen_cpp
-THRIFT_JAVA_ARGS = -I ${CURDIR} -I ${BUILD_DIR}/thrift/ --gen java -out ${BUILD_DIR}/java
 
 ${BUILD_DIR}/gen_cpp:
 	mkdir -p $@
@@ -43,14 +39,3 @@ ${BUILD_DIR}/gen_cpp/%_types.cpp: ${CURDIR}/%.thrift | ${BUILD_DIR}/gen_cpp
 # generated thrift
 ${BUILD_DIR}/gen_cpp/%_types.cpp: ${BUILD_DIR}/thrift/%.thrift | ${BUILD_DIR}/gen_cpp
 	${THRIFT} ${THRIFT_CPP_ARGS} $<
-
-# generate java object
-# TODO(zhaochun): make depends generated java 
-# handwrite thrift
-${BUILD_DIR}/java/.%.tm: ${CURDIR}/%.thrift | ${BUILD_DIR}/java
-	${THRIFT} ${THRIFT_JAVA_ARGS} $<
-	touch $@
-# generated thrift
-${BUILD_DIR}/java/.%.tm: ${BUILD_DIR}/thrift/%.thrift | ${BUILD_DIR}/java
-	${THRIFT} ${THRIFT_JAVA_ARGS} $<
-	touch $@