diff --git a/build.sh b/build.sh index 45b599f9d87a5e..577ef81f10be77 100755 --- a/build.sh +++ b/build.sh @@ -41,8 +41,7 @@ if [[ ! -f ${DORIS_THIRDPARTY}/installed/lib/libs2.a ]]; then ${DORIS_THIRDPARTY}/build-thirdparty.sh fi -#PARALLEL=$[$(nproc)/4+1] -PARALLEL=12 +PARALLEL=$[$(nproc)/4+1] # Check args usage() { @@ -148,6 +147,7 @@ echo "Build generated code" cd ${DORIS_HOME}/gensrc if [ ${CLEAN} -eq 1 ]; then make clean + rm -rf ${DORIS_HOME}/fe/fe-core/target fi # DO NOT using parallel make(-j) for gensrc make diff --git a/docs/.vuepress/public/images/DEBUG1.png b/docs/.vuepress/public/images/DEBUG1.png deleted file mode 100644 index a565adbd97fe13..00000000000000 Binary files a/docs/.vuepress/public/images/DEBUG1.png and /dev/null differ diff --git a/docs/.vuepress/public/images/DEBUG2.png b/docs/.vuepress/public/images/DEBUG2.png deleted file mode 100644 index 994b27494fcb40..00000000000000 Binary files a/docs/.vuepress/public/images/DEBUG2.png and /dev/null differ diff --git a/docs/.vuepress/public/images/DEBUG3.png b/docs/.vuepress/public/images/DEBUG3.png deleted file mode 100644 index f9cde69dfe74b6..00000000000000 Binary files a/docs/.vuepress/public/images/DEBUG3.png and /dev/null differ diff --git a/docs/.vuepress/public/images/gen_code.png b/docs/.vuepress/public/images/gen_code.png new file mode 100644 index 00000000000000..7b328417418440 Binary files /dev/null and b/docs/.vuepress/public/images/gen_code.png differ diff --git a/docs/en/developer-guide/fe-eclipse-dev.md b/docs/en/developer-guide/fe-eclipse-dev.md index c4834be0ec02e8..00b8df151c32e7 100644 --- a/docs/en/developer-guide/fe-eclipse-dev.md +++ b/docs/en/developer-guide/fe-eclipse-dev.md @@ -34,23 +34,23 @@ under the License. ### Code Generation -The FE module requires part of the generated code, such as Thrift, Protobuf and other frameworks. This part needs to be generated by compiling the contents of the `gensrc/` directory in the Linux environment. +The FE module requires part of the generated code, such as Thrift, Protobuf, Jflex, CUP and other frameworks. -1. Under Linux, enter the source code directory and execute the following command: +1. Under Linux, enter the source code directory `fe` and execute the following command: - ``` - sh build.sh --clean --fe - ``` + ``` + mvn generate-sources + ``` -2. Then package the generated `gensrc/build/java` directory: +2. If use windows as development environment, then package the generated `fe/fe-core/target/generated-sources` directory: - `cd gensrc/build/ && tar czf java.tar.gz java/` + `fe/fe-core/target/ && tar czf java.tar.gz generated-sources/` -3. Copy `java.tar.gz` to the `fe/` directory of the development environment and unzip +3. Copy `java.tar.gz` to the `fe/fe-core/target/` directory of the development environment and unzip ``` - cp java.tar.gz /path/to/doris/fe/ - cd /path/to/doris/fe/ && tar xzf java.tar.gz + cp java.tar.gz /path/to/doris/fe/fe-core/target/ + cd /path/to/doris/fe/fe-core/target/ && tar xzf java.tar.gz ``` ## Import FE project @@ -126,13 +126,12 @@ You can directly start an FE process in Ecplise to facilitate debugging the code ## Code Update -1. Update lexical and grammar files +1. Update lexical and grammar files or proto and thrift files - If you modified `fe/src/main/cup/sql_parser.cup` or `fe/src/main/jflex/sql_scanner.flex` file. You need to execute the following commands in the `fe/` directory: + If you modified `fe/src/main/cup/sql_parser.cup` or `fe/src/main/jflex/sql_scanner.flex` file or proto and thrift files. You need to execute the following commands in the `fe/` directory: ``` - mvn de.jflex:maven-jflex-plugin:1.4.3:generate - mvn net.sourceforge.czt.dev:cup-maven-plugin:1.6-cdh:generate + mvn generate-sources ``` Then refresh the project in Eclipse. diff --git a/docs/zh-CN/developer-guide/fe-eclipse-dev.md b/docs/zh-CN/developer-guide/fe-eclipse-dev.md index 25c01321b281b0..198614463089a0 100644 --- a/docs/zh-CN/developer-guide/fe-eclipse-dev.md +++ b/docs/zh-CN/developer-guide/fe-eclipse-dev.md @@ -34,23 +34,22 @@ under the License. ### 代码生成 -FE 模块需要部分生成代码，如 Thrift、Protobuf 等框架的生成代码。这部分需要在 Linux 环境中通过编译 `gensrc/` 目录中的内容生成。 +FE 模块需要部分生成代码，如 Thrift、Protobuf, jflex, cup 等框架的生成代码。这部分需要在 Linux 或者 Mac环境生成。 -1. 在 Linux 下，进入到源码目录，执行下面的命令： - - ``` - sh build.sh --clean --fe - ``` +1. 在 Linux 下， 进入 `fe 目录下执行以下命令： + ``` + mvn generate-sources + ``` -2. 然后将生成的 `gensrc/build/java` 目录打包： +2. 如果使用window开发 需要将生成的 `fe/fe-core/target/generated-sources` 目录打包： - `cd gensrc/build/ && tar czf java.tar.gz java/` + `fe/fe-core/target/ && tar czf java.tar.gz generated-sources/` -3. 将 `java.tar.gz` 拷贝到开发环境的 `fe/` 目录下，并解压 +3. 将 `java.tar.gz` 拷贝到开发环境的 `fe/fe-core/target/` 目录下，并解压 ``` - cp java.tar.gz /path/to/doris/fe/ - cd /path/to/doris/fe/ && tar xzf java.tar.gz + cp java.tar.gz /path/to/doris/fe/fe-core/target/ + cd /path/to/doris/fe/fe-core/target/ && tar xzf java.tar.gz ``` ## 导入 FE 工程 @@ -126,13 +125,12 @@ java.lang.Exception: Method xxxx should have no parameters ## 代码更新 -1. 更新词法、语法文件 +1. 更新词法、语法文件或者thrift 和proto 文件 - 如果修改了 `fe/src/main/cup/sql_parser.cup` 或者 `fe/src/main/jflex/sql_scanner.flex` 文件。则需在 `fe/` 目录下执行以下命令： + 如果修改了 `fe/fe-core/src/main/cup/sql_parser.cup` 或者 `fe/fe-core/src/main/jflex/sql_scanner.flex`文件或者proto 和thrift 文件。则需在 `fe` 目录下执行以下命令： ``` - mvn de.jflex:maven-jflex-plugin:1.4.3:generate - mvn net.sourceforge.czt.dev:cup-maven-plugin:1.6-cdh:generate + mvn generate-sources ``` 之后在 Eclipse 中刷新工程即可。 diff --git a/docs/zh-CN/developer-guide/fe-idea-dev.md b/docs/zh-CN/developer-guide/fe-idea-dev.md index ad1bf1541f1cdc..e511ed3ae0eac8 100644 --- a/docs/zh-CN/developer-guide/fe-idea-dev.md +++ b/docs/zh-CN/developer-guide/fe-idea-dev.md @@ -30,27 +30,17 @@ under the License. JDK1.8+ , Intellj IDEA -1.linux上编译好fe前端代码，主要目的是获取自动生成的代码，加入到前段工程里面去用于在idea中编译fe工程 - -在linux下，进入到源码目录，执行下面的命令： - +1. 从 https://github.com/apache/incubator-doris.git 下载源码到本地 +2. 使用Intellj IDEA 打开代码根目录 +3. 如果仅进行fe开发而没有编译过thirdparty，则需要安装thrift，并将thrift 复制或者连接到 `thirdparty/installed/bin` 目录下 +4. 如果是Mac 或者 Linux 环境 可以通过 如下命令生成自动生成代码： ``` -$ sh build.sh --clean --fe +cd fe +mvn generate-sources ``` - -然后将 gensrc目录打包，拿出来，如下图 - -![](/images/DEBUG1.png) - -2.在windows下解压gensrc.tar.gz,解压后的目录如下图： - -![](/images/DEBUG2.png) - -3.进入build/java,将红色框出的部分整个拷贝到源码的fe/src/main/java目录下 - - - -![](/images/DEBUG3.png) +或者通过图形界面运行运行maven 命令生成 +![](/images/gen_code.png) +如果使用windows环境可能会有make命令和sh脚本无法执行的情况 可以通过拷贝linux上的 `fe/fe-core/target/generated-sources` 目录拷贝到相应的目录的方式实现，也可以通过docker 镜像挂载本地目录之后，在docker 内部生成自动生成代码，可以参照编译一节 ## 2.调试 diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index 7e9518adc8119f..f3a4b9247b6df1 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -20,7 +20,7 @@ under the License. --> + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 @@ -37,8 +37,23 @@ under the License. ${basedir}/../../ ${env.FE_UT_PARALLEL} + ${basedir}/../../thirdparty + + + thirdparty + + + env.DORIS_THIRDPARTY + + + + ${env.DORIS_THIRDPARTY} + + + + @@ -428,7 +443,7 @@ under the License. spark-launcher_2.12 - + org.apache.spark spark-sql_2.12 @@ -440,6 +455,25 @@ under the License. palo-fe + + + org.apache.thrift.tools + maven-thrift-plugin + 0.1.11 + + ${doris.thridparty}/installed/bin/thrift + ${doris.home}/gensrc/thrift + + + + thrift-sources + generate-sources + + compile + + + + net.sourceforge.czt.dev @@ -506,8 +540,23 @@ under the License. org.codehaus.mojo exec-maven-plugin - 1.6.0 + 3.0.0 + + make-dir + generate-sources + + exec + + + mkdir + + -p + ${basedir}/target/generated-sources/proto + + ${skip.plugin} + + gensrc generate-sources @@ -518,7 +567,7 @@ under the License. make -C - ${doris.home}/gensrc/ + ${doris.home}/gensrc/script ${skip.plugin} @@ -535,7 +584,7 @@ under the License. -jar ${settings.localRepository}/com/baidu/jprotobuf/${jprotobuf.version}/jprotobuf-${jprotobuf.version}-jar-with-dependencies.jar - --java_out=${doris.home}/gensrc/build/java/ + --java_out=${basedir}/target/generated-sources/proto ${doris.home}/gensrc/proto/internal_service.proto ${skip.plugin} @@ -559,7 +608,8 @@ under the License. - ${doris.home}/gensrc/build/java/ + ${basedir}/target/generated-sources/build/ + ${basedir}/target/generated-sources/proto/ diff --git a/gensrc/script/Makefile b/gensrc/script/Makefile index 75af53bdd21bb9..c824478cf3a70c 100644 --- a/gensrc/script/Makefile +++ b/gensrc/script/Makefile @@ -18,6 +18,7 @@ # generate files with python BUILD_DIR = ${CURDIR}/../build/ +FE_TARGET_DIR = ${CURDIR}/../../fe/fe-core/target/generated-sources/build # Prerequisites on the right side of '|' is only order all: gen_builtins gen_version @@ -45,7 +46,7 @@ gen_vec_func: ${GEN_VEC_FUNC_OUTPUT} .PHONY: gen_vec_func # generate -GEN_BUILTINS_OUTPUT = ${BUILD_DIR}/java/org/apache/doris/builtins/ScalarBuiltins.java +GEN_BUILTINS_OUTPUT = ${FE_TARGET_DIR}/org/apache/doris/builtins/ScalarBuiltins.java ${GEN_BUILTINS_OUTPUT}: doris_builtins_functions.py gen_builtins_functions.py ${GEN_FUNC_OUTPUT} ${GEN_VEC_FUNC_OUTPUT} cd ${BUILD_DIR}/python && ${PYTHON} ${CURDIR}/gen_builtins_functions.py diff --git a/gensrc/script/gen_build_version.sh b/gensrc/script/gen_build_version.sh index 21bb1b89419344..1cf5420bff3829 100755 --- a/gensrc/script/gen_build_version.sh +++ b/gensrc/script/gen_build_version.sh @@ -45,7 +45,7 @@ then fi if [[ -z ${DORIS_TEST_BINARY_DIR} ]]; then - if [ -e ${DORIS_HOME}/gensrc/build/java/org/apache/doris/common/Version.java \ + if [ -e ${DORIS_HOME}/fe/fe-core/target/generated-sources/build/org/apache/doris/common/Version.java \ -a -e ${DORIS_HOME}/gensrc/build/gen_cpp/version.h ]; then exit fi @@ -91,7 +91,7 @@ java_version_str=$(echo $java_version_str | sed -e 's/"/\\"/g') echo "get java cmd: $java_cmd" echo "get java version: $java_version_str" -VERSION_PACKAGE="${DORIS_HOME}/gensrc/build/java/org/apache/doris/common" +VERSION_PACKAGE="${DORIS_HOME}/fe/fe-core/target/generated-sources/build/org/apache/doris/common" mkdir -p ${VERSION_PACKAGE} cat >"${VERSION_PACKAGE}/Version.java" <