Merge branch 'master' of https://github.com/vesoft-inc/nebula into ne…

…bula-breakpad
vesoft-inc · Mar 8, 2022 · 371c52d · 371c52d
2 parents bb74be1 + 8e5a30a
commit 371c52d
Show file tree

Hide file tree

Showing 854 changed files with 19,890 additions and 12,652 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -0,0 +1,6 @@
+#Require an approved review in PRs including files with a designated code owner.
+/conf/ @vesoft-inc/tech-committee-reviewers
+/src/kvstore/raftex/ @critical27 @sherman-the-tank
+/cmake/ @sherman-the-tank @yixinglu @dutor
+*.thrift @vesoft-inc/tech-committee-reviewers
+*.yy @CPWstatic @dutor
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -3,26 +3,26 @@ Thanks for your contribution!
 In order to review PR more efficiently, please add information according to the template.
 -->
 
-#### What type of PR is this?
+## What type of PR is this?
 - [ ] bug
 - [ ] feature
 - [ ] enhancement
 
-#### What problem(s) does this PR solve?
-Issue(s) number: 
+## What problem(s) does this PR solve?
+#### Issue(s) number: 
 
-Description:
+#### Description:
 
 
-#### How do you solve it?
+## How do you solve it?
 
 
-
-#### Special notes for your reviewer, ex. impact of this fix, design document, etc:
 
+## Special notes for your reviewer, ex. impact of this fix, design document, etc:
 
 
-#### Checklist:
+
+## Checklist:
 Tests:
 - [ ] Unit test(positive and negative cases)
 - [ ] Function test
@@ -36,7 +36,7 @@ Affects:
 - [ ] Performance impacted: Consumes more CPU/Memory
 
 
-#### Release notes:
+## Release notes:
 
 Please confirm whether to be reflected in release notes and how to describe:
 > ex. Fixed the bug .....
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -33,6 +33,9 @@ jobs:
         with:
           run: sh -c "find . -mindepth 1 -delete"
       - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16.7'
       - name: package
         run: ./package/package.sh
       - name: output some vars
@@ -82,8 +85,7 @@ jobs:
         with:
           context: .
           file: ./docker/Dockerfile.${{ matrix.service }}
-          # platforms: linux/amd64,linux/arm64
-          platforms: linux/amd64
+          platforms: linux/amd64,linux/arm64
           tags: |
             vesoft/nebula-${{ matrix.service }}:nightly
           push: true

diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
@@ -170,6 +170,11 @@ jobs:
           make RM_DIR=false DEBUG=false J=${{ steps.cmake.outputs.j }} tck
         working-directory: tests/
         timeout-minutes: 60
+      - name: LDBC
+        run: |
+          make RM_DIR=false DEBUG=false J=${{ steps.cmake.outputs.j }} ldbc
+        working-directory: tests/
+        timeout-minutes: 60
       - name: Down cluster
         run: |
           make RM_DIR=false down
@@ -194,3 +199,92 @@ jobs:
         with:
           name: ${{ matrix.os }}-${{ matrix.compiler }}-nebula-test-logs
           path:  ./build/server_*/logs/
+
+  standalone:
+    name: standalone-build
+    needs: lint
+    runs-on: [self-hosted, nebula]
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - centos7
+        compiler:
+          - gcc-9.3
+    env:
+      CCACHE_DIR: /tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }}
+      CCACHE_MAXSIZE: 8G
+    container:
+      image: vesoft/nebula-dev:${{ matrix.os }}
+      volumes:
+        - /tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }}:/tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }}
+      options: --cap-add=SYS_PTRACE
+    steps:
+      - uses: webiny/action-post-run@2.0.1
+        with:
+          run: sh -c "find . -mindepth 1 -delete"
+      - uses: actions/checkout@v2
+      - name: Prepare environment
+        id: prepare
+        run: |
+          [ -d build/ ] && rm -rf build/* || mkdir -p build
+          make init -C tests
+      - name: CMake
+        id: cmake
+        run: |
+          case ${{ matrix.compiler }} in
+          gcc-*)
+            case ${{ matrix.os }} in
+            centos7)
+              # build with Release type
+              cmake \
+                  -DCMAKE_CXX_COMPILER=$TOOLSET_GCC_DIR/bin/g++ \
+                  -DCMAKE_C_COMPILER=$TOOLSET_GCC_DIR/bin/gcc \
+                  -DCMAKE_BUILD_TYPE=Release \
+                  -DENABLE_TESTING=on \
+                  -DENABLE_STANDALONE_VERSION=on \
+                  -GNinja \
+                  -B build
+              echo "::set-output name=j::10"
+              ;;
+            esac
+            ;;
+          esac
+      - name: Make
+        run: |
+          ccache -z
+          ninja -j $(nproc)
+          ccache -s
+        working-directory: build/
+      - name: CTest
+        env:
+          ASAN_OPTIONS: fast_unwind_on_malloc=1
+        run: ctest -j $(($(nproc)/2+1)) --timeout 400 --output-on-failure
+        working-directory: build/
+        timeout-minutes: 20
+      - name: Setup Cluster
+        run: |
+          make standalone-up
+        working-directory: tests/
+        timeout-minutes: 60
+      - name: TCK
+        run: |
+          make RM_DIR=false DEBUG=false J=${{ steps.cmake.outputs.j }} standalone-tck
+        working-directory: tests/
+        timeout-minutes: 60
+      - name: LDBC
+        run: |
+          make RM_DIR=false DEBUG=false J=${{ steps.cmake.outputs.j }} ldbc
+        working-directory: tests/
+        timeout-minutes: 60
+      - name: Down cluster
+        run: |
+          make RM_DIR=false down
+        working-directory: tests/
+        timeout-minutes: 2
+      - name: Upload logs
+        uses: actions/upload-artifact@v2
+        if: ${{ failure() }}
+        with:
+          name: ${{ matrix.os }}-${{ matrix.compiler }}-nebula-test-logs
+          path:  ./build/server_*/logs/
diff --git a/.github/workflows/rc.yml b/.github/workflows/rc.yml
@@ -37,10 +37,13 @@ jobs:
         with:
           run: sh -c "find . -mindepth 1 -delete"
       - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '^1.16.7'
       - uses: ./.github/actions/tagname-action
         id: tag
       - name: package
-        run: ./package/package.sh -v ${{ steps.tag.outputs.tag }} -t RelWithDebInfo -r OFF -p ON -s TRUE -k ON
+        run: ./package/package.sh -v ${{ steps.tag.outputs.tagnum }} -t RelWithDebInfo -r OFF -p ON -s TRUE -k ON
       - name: output some vars
         run: |
           tar zcf ${{ env.CPACK_DIR }}/nebula-${{ steps.tag.outputs.tagnum }}.tar.gz --exclude=${{ env.BUILD_DIR }} ./*
@@ -99,12 +102,88 @@ jobs:
         with:
           context: .
           file: ./docker/Dockerfile.${{ matrix.service }}
-          # platforms: linux/amd64,linux/arm64
-          platforms: linux/amd64
+          platforms: linux/amd64,linux/arm64
           tags: |
             ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-${{ matrix.service }}:${{ steps.tagname.outputs.tag }}
             ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-${{ matrix.service }}:${{ steps.tagname.outputs.majorver }}
             ${{ steps.docker.outputs.tag }}
           push: true
           build-args: |
             BRANCH=${{ steps.tagname.outputs.tag }}
+
+  test:
+    name: test
+    runs-on: [self-hosted, nebula]
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu1804
+          - ubuntu2004
+          - centos7
+          - centos8
+        compiler:
+          - gcc-9.3
+    container:
+      image: vesoft/nebula-dev:${{ matrix.os }}
+      env:
+        CCACHE_DIR: /tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }}
+        CCACHE_MAXSIZE: 8G
+      volumes:
+        - /tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }}:/tmp/ccache/nebula/${{ matrix.os }}-${{ matrix.compiler }}
+      options: --cap-add=SYS_PTRACE
+    steps:
+      - uses: webiny/action-post-run@2.0.1
+        with:
+          run: sh -c "find . -mindepth 1 -delete"
+      - uses: actions/checkout@v2
+      - name: Prepare environment
+        id: prepare
+        run: |
+          [ -d build/ ] && rm -rf build/* || mkdir -p build
+          make init -C tests
+      - name: CMake
+        id: cmake
+        run: |
+          cmake \
+              -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+              -DENABLE_TESTING=on \
+              -B build
+          echo "::set-output name=j::8"
+      - name: Make
+        run: |
+          ccache -z
+          cmake --build build/ -j $(nproc)
+          ccache -s
+      - name: CTest
+        env:
+          ASAN_OPTIONS: fast_unwind_on_malloc=1
+        run: ctest -j $(($(nproc)/2+1)) --timeout 400 --output-on-failure
+        working-directory: build/
+        timeout-minutes: 20
+      - name: Setup cluster
+        run: |
+          make CONTAINERIZED=true ENABLE_SSL=true CA_SIGNED=true up
+        working-directory: tests/
+        timeout-minutes: 2
+      - name: Pytest
+        run: |
+          make RM_DIR=false DEBUG=false J=${{ steps.cmake.outputs.j }} test
+        working-directory: tests/
+        timeout-minutes: 15
+      - name: TCK
+        run: |
+          make RM_DIR=false DEBUG=false J=${{ steps.cmake.outputs.j }} tck
+        working-directory: tests/
+        timeout-minutes: 60
+      - name: Down cluster
+        run: |
+          make RM_DIR=false down
+        working-directory: tests/
+        timeout-minutes: 2
+      - name: Upload logs
+        uses: actions/upload-artifact@v2
+        if: ${{ failure() }}
+        with:
+          name: ${{ matrix.os }}-${{ matrix.compiler }}-nebula-test-logs
+          path:  ./build/server_*/logs/
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -61,7 +61,7 @@ jobs:
         env:
           FROM_IMAGE: docker://${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-${{ matrix.service }}
           TO_IMAGE: docker://docker.io/vesoft/nebula-${{ matrix.service }}
-          CMD: docker run --rm -ti quay.io/containers/skopeo:v1.4.1 copy -a --src-creds ${{ secrets.HARBOR_USERNAME }}:${{ secrets.HARBOR_PASSWORD }} --dest-creds ${{ secrets.DOCKER_USERNAME }}:${{ secrets.DOCKER_PASSWORD }}
+          CMD: docker run --rm -i quay.io/containers/skopeo:v1.4.1 copy -a --src-creds ${{ secrets.HARBOR_USERNAME }}:${{ secrets.HARBOR_PASSWORD }} --dest-creds ${{ secrets.DOCKER_USERNAME }}:${{ secrets.DOCKER_PASSWORD }}
         run: |
           ${{ env.CMD }} ${{ env.FROM_IMAGE }}:${{ steps.tagname.outputs.tag }} ${{ env.TO_IMAGE }}:${{ steps.tagname.outputs.tag }}
           ${{ env.CMD }} ${{ env.FROM_IMAGE }}:${{ steps.tagname.outputs.tag }} ${{ env.TO_IMAGE }}:${{ steps.tagname.outputs.majorver }}

diff --git a/.linters/cpp/cpplint.py b/.linters/cpp/cpplint.py
@@ -2130,9 +2130,11 @@ def CheckForHeaderGuard(filename, clean_lines, error):
     if Search(r'//\s*NOLINT\(build/header_guard\)', i):
       return
 
-  # Allow pragma once instead of header guards
+  # notallow pragma once
   for i in raw_lines:
     if Search(r'^\s*#pragma\s+once', i):
+      error(filename, 0, 'build/header_guard', 5,
+        'Pragma once not allowed, please use ifndef Instead')
       return
 
   cppvar = GetHeaderGuardCPPVariable(filename)

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -25,6 +25,7 @@ project("Nebula Graph" C CXX)
 
 option(ENABLE_PACK_ONE "Whether to package into one" ON)
 option(ENABLE_VERBOSE_BISON "Enable Bison to report state" OFF)
+option(ENABLE_CONSOLE_COMPILATION "Enable nebula-console compilation" OFF)
 option(ENABLE_PACKAGE_TAR "Enable package artifacts to tar." OFF)
 option(ENABLE_CREATE_GIT_HOOKS "Enable create git hooks." ON)
 option(ENABLE_INCLUDE_WHAT_YOU_USE "Enable include-what-you-use find  nouse include files" OFF)
@@ -52,6 +53,7 @@ include(GitInfoConfig)
 include(NebulaCustomTargets)
 include(IncludeWhatYouUse)
 include(BreakPadConfig)
+include(TimeTrace)
 
 add_custom_target(
   clang-format

diff --git a/cmake/CPackage.cmake b/cmake/CPackage.cmake
@@ -162,9 +162,11 @@ macro(package to_one name home_page scripts_dir)
     cpack_add_component(storage GROUP storage DEPENDS common)
     cpack_add_component(meta GROUP meta DEPENDS common)
     cpack_add_component(tool GROUP tool)
+    cpack_add_component(nebula-console GROUP nebula-console)
     cpack_add_component_group(common)
     cpack_add_component_group(graph)
     cpack_add_component_group(storage)
     cpack_add_component_group(meta)
     cpack_add_component_group(tool)
+    cpack_add_component_group(nebula-console)
 endmacro()
diff --git a/cmake/ThriftGenerate.cmake b/cmake/ThriftGenerate.cmake
@@ -86,7 +86,7 @@ add_custom_command(
     --gen "js:node:"
     --gen "csharp"
     --gen "java:hashcode"
-    --gen "go:thrift_import=github.com/facebook/fbthrift/thrift/lib/go/thrift,package_prefix=github.com/vesoft-inc/nebula-go/v2/,use_context"
+    --gen "go:thrift_import=github.com/facebook/fbthrift/thrift/lib/go/thrift,package_prefix=github.com/vesoft-inc/nebula-go/v3/,use_context"
     -o "." "${file_path}/${file_name}.thrift"
   COMMAND
     mkdir -p "./gen-rust/${file_name}"

diff --git a/cmake/nebula/TimeTrace.cmake b/cmake/nebula/TimeTrace.cmake
@@ -0,0 +1,6 @@
+if(CMAKE_CXX_COMPILER_ID MATCHES ".*Clang")
+  option(ENABLE_BUILD_WITH_TIME_TRACE "Enable -ftime-trace to generate time tracing .json files on clang" OFF)
+  if(ENABLE_BUILD_WITH_TIME_TRACE)
+    add_compile_options(-ftime-trace)
+  endif()
+endif()
diff --git a/conf/nebula-graphd.conf.default b/conf/nebula-graphd.conf.default
@@ -67,8 +67,6 @@
 --ws_ip=0.0.0.0
 # HTTP service port
 --ws_http_port=19669
-# HTTP2 service port
---ws_h2_port=19670
 # storage client timeout
 --storage_client_timeout_ms=60000
 # Port to listen on Meta with HTTP protocol, it corresponds to ws_http_port in metad's configuration file
@@ -84,8 +82,9 @@
 # System memory high watermark ratio, cancel the memory checking when the ratio greater than 1.0
 --system_memory_high_watermark_ratio=0.8
 
+########## metrics ##########
+--enable_space_level_metrics=false
+
 ########## experimental feature ##########
 # if use experimental features
 --enable_experimental_feature=false
-
---enable_space_level_metrics=false
diff --git a/conf/nebula-graphd.conf.production b/conf/nebula-graphd.conf.production
@@ -66,8 +66,6 @@
 --ws_ip=0.0.0.0
 # HTTP service port
 --ws_http_port=19669
-# HTTP2 service port
---ws_h2_port=19670
 # storage client timeout
 --storage_client_timeout_ms=60000
 # Port to listen on Meta with HTTP protocol, it corresponds to ws_http_port in metad's configuration file
@@ -83,8 +81,9 @@
 # System memory high watermark ratio, cancel the memory checking when the ratio greater than 1.0
 --system_memory_high_watermark_ratio=0.8
 
+########## metrics ##########
+--enable_space_level_metrics=false
+
 ########## experimental feature ##########
 # if use experimental features
 --enable_experimental_feature=false
-
---enable_space_level_metrics=false
diff --git a/conf/nebula-metad.conf.default b/conf/nebula-metad.conf.default
@@ -36,8 +36,6 @@
 --ws_ip=0.0.0.0
 # HTTP service port
 --ws_http_port=19559
-# HTTP2 service port
---ws_h2_port=19560
 # Port to listen on Storage with HTTP protocol, it corresponds to ws_http_port in storage's configuration file
 --ws_storage_http_port=19779