Merge branch 'master' into fix/remove-debug-code

.github/workflows/nightly.yml

@@ -62,6 +62,14 @@ jobs:
  docker:
  name: build docker image
  runs-on: [self-hosted, nebula]
+ strategy:
+ fail-fast: false
+ matrix:
+ service:
+ - graphd
+ - metad
+ - storaged
+ - tools
  steps:
  - uses: webiny/action-post-run@2.0.1
  with:
@@ -76,38 +84,10 @@ jobs:
  - uses: docker/build-push-action@v2
  with:
  context: .
- file: ./docker/Dockerfile
- platforms: linux/amd64,linux/arm64
- tags: |
- vesoft/nebula-graphd:nightly
- target: graphd
- push: true
- - uses: docker/build-push-action@v2
- with:
- context: .
- file: ./docker/Dockerfile
- platforms: linux/amd64,linux/arm64
- tags: |
- vesoft/nebula-storaged:nightly
- target: storaged
- push: true
- - uses: docker/build-push-action@v2
- with:
- context: .
- file: ./docker/Dockerfile
- platforms: linux/amd64,linux/arm64
- tags: |
- vesoft/nebula-metad:nightly
- target: metad
- push: true
- - uses: docker/build-push-action@v2
- with:
- context: .
- file: ./docker/Dockerfile
+ file: ./docker/Dockerfile.${{ matrix.service }}
  platforms: linux/amd64,linux/arm64
  tags: |
- vesoft/nebula-tools:nightly
- target: tools
+ vesoft/nebula-${{ matrix.service }}:nightly
  push: true
 
  coverage:

.github/workflows/rc.yml

@@ -68,6 +68,14 @@ jobs:
  docker_build:
  name: docker-build
  runs-on: [self-hosted, nebula]
+ strategy:
+ fail-fast: false
+ matrix:
+ service:
+ - graphd
+ - metad
+ - storaged
+ - tools
  steps:
  - uses: webiny/action-post-run@2.0.1
  with:
@@ -93,62 +101,16 @@ jobs:
  - uses: docker/build-push-action@v2
  with:
  context: .
- file: ./docker/Dockerfile
- platforms: linux/amd64,linux/arm64
- tags: |
- ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-graphd:${{ steps.tagname.outputs.tag }}
- ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-graphd:${{ steps.tagname.outputs.majorver }}
- ${{ steps.docker.outputs.tag }}
- push: true
- target: graphd
- build-args: |
- BRANCH=${{ steps.tagname.outputs.tag }}
- VERSION=${{ steps.tagname.outputs.tagnum }}
-
- - uses: docker/build-push-action@v2
- with:
- context: .
- file: ./docker/Dockerfile
- platforms: linux/amd64,linux/arm64
- tags: |
- ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-storaged:${{ steps.tagname.outputs.tag }}
- ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-storaged:${{ steps.tagname.outputs.majorver }}
- ${{ steps.docker.outputs.tag }}
- push: true
- target: storaged
- build-args: |
- BRANCH=${{ steps.tagname.outputs.tag }}
- VERSION=${{ steps.tagname.outputs.tagnum }}
-
- - uses: docker/build-push-action@v2
- with:
- context: .
- file: ./docker/Dockerfile
- platforms: linux/amd64,linux/arm64
- tags: |
- ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-metad:${{ steps.tagname.outputs.tag }}
- ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-metad:${{ steps.tagname.outputs.majorver }}
- ${{ steps.docker.outputs.tag }}
- push: true
- target: metad
- build-args: |
- BRANCH=${{ steps.tagname.outputs.tag }}
- VERSION=${{ steps.tagname.outputs.tagnum }}
- 
- - uses: docker/build-push-action@v2
- with:
- context: .
- file: ./docker/Dockerfile
+ file: ./docker/Dockerfile.${{ matrix.service }}
  platforms: linux/amd64,linux/arm64
  tags: |
- ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-tools:${{ steps.tagname.outputs.tag }}
- ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-tools:${{ steps.tagname.outputs.majorver }}
+ ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-${{ matrix.service }}:${{ steps.tagname.outputs.tag }}
+ ${{ secrets.HARBOR_REGISTRY }}/vesoft/nebula-${{ matrix.service }}:${{ steps.tagname.outputs.majorver }}
  ${{ steps.docker.outputs.tag }}
  push: true
- target: tools
  build-args: |
  BRANCH=${{ steps.tagname.outputs.tag }}
- VERSION=${{ steps.tagname.outputs.tagnum }}
+ VERSON=${{ steps.tagname.outputs.tagnum }}
 
  test:
  name: test

docker/Dockerfile

@@ -2,6 +2,7 @@ FROM vesoft/nebula-dev:centos7 as builder
 
 ARG BRANCH=master
 ARG VERSION=
+
 COPY . /home/nebula/BUILD
 
 RUN cd /home/nebula/BUILD/package \

docker/Dockerfile.graphd

@@ -0,0 +1,26 @@
+FROM vesoft/nebula-dev:centos7 as builder
+
+ARG BRANCH=master
+ARG VERSION=
+
+COPY . /home/nebula/BUILD
+
+RUN cd /home/nebula/BUILD/package \
+ && [[ -z "${VERSION}" ]] \
+ && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON \
+ || ./package.sh -n OFF -v ${VERSION} -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON 
+
+FROM centos:7
+
+COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-common.rpm /usr/local/nebula/nebula-common.rpm
+COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-graph.rpm /usr/local/nebula/nebula-graphd.rpm
+
+WORKDIR /usr/local/nebula
+
+RUN rpm -ivh *.rpm \
+ && mkdir -p ./{logs,data,pids} \
+ && rm -rf *.rpm
+
+EXPOSE 9669 19669 19670
+
+ENTRYPOINT ["/usr/local/nebula/bin/nebula-graphd", "--flagfile=/usr/local/nebula/etc/nebula-graphd.conf", "--daemonize=false", "--containerized=true"]

docker/Dockerfile.metad

@@ -0,0 +1,26 @@
+FROM vesoft/nebula-dev:centos7 as builder
+
+ARG BRANCH=master
+ARG VERSION=
+
+COPY . /home/nebula/BUILD
+
+RUN cd /home/nebula/BUILD/package \
+ && [[ -z "${VERSION}" ]] \
+ && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON \
+ || ./package.sh -n OFF -v ${VERSION} -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON 
+
+FROM centos:7
+
+COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-common.rpm /usr/local/nebula/nebula-common.rpm
+COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-meta.rpm /usr/local/nebula/nebula-metad.rpm
+
+WORKDIR /usr/local/nebula
+
+RUN rpm -ivh *.rpm \
+ && mkdir -p ./{logs,data,pids} \
+ && rm -rf *.rpm
+
+EXPOSE 9559 9560 19559 19560
+
+ENTRYPOINT ["/usr/local/nebula/bin/nebula-metad", "--flagfile=/usr/local/nebula/etc/nebula-metad.conf", "--daemonize=false", "--containerized=true"]

docker/Dockerfile.storaged

@@ -0,0 +1,26 @@
+FROM vesoft/nebula-dev:centos7 as builder
+
+ARG BRANCH=master
+ARG VERSION=
+
+COPY . /home/nebula/BUILD
+
+RUN cd /home/nebula/BUILD/package \
+ && [[ -z "${VERSION}" ]] \
+ && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON \
+ || ./package.sh -n OFF -v ${VERSION} -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON 
+
+FROM centos:7
+
+COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-common.rpm /usr/local/nebula/nebula-common.rpm
+COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-storage.rpm /usr/local/nebula/nebula-storaged.rpm
+
+WORKDIR /usr/local/nebula
+
+RUN rpm -ivh *.rpm \
+ && mkdir -p ./{logs,data,pids} \
+ && rm -rf *.rpm
+
+EXPOSE 9777 9778 9779 9780 19779 19780
+
+ENTRYPOINT ["/usr/local/nebula/bin/nebula-storaged", "--flagfile=/usr/local/nebula/etc/nebula-storaged.conf", "--daemonize=false", "--containerized=true"]

docker/Dockerfile.tools

@@ -0,0 +1,23 @@
+FROM vesoft/nebula-dev:centos7 as builder
+
+ARG BRANCH=master
+ARG VERSION=
+
+COPY . /home/nebula/BUILD
+
+RUN cd /home/nebula/BUILD/package \
+ && [[ -z "${VERSION}" ]] \
+ && ./package.sh -n OFF -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON \
+ || ./package.sh -n OFF -v ${VERSION} -b ${BRANCH} -t RelWithDebInfo -s TRUE -c OFF -k ON 
+
+FROM centos:7
+
+COPY --from=builder /home/nebula/BUILD/pkg-build/cpack_output/nebula-*-tool.rpm /usr/local/nebula/nebula-tool.rpm
+
+WORKDIR /usr/local/nebula
+
+RUN rpm -ivh *.rpm \
+ && rm -rf *.rpm
+
+# default entrypoint
+ENTRYPOINT ["/usr/local/nebula/bin/db_dump"]

docker/README.md

@@ -2,7 +2,7 @@
 
 Following docker images will be ready in production.
 
-- [vesoft/nebula-graphd](https://hub.docker.com/r/vesoft/nebula-graphd): nebula-graphd service built with `Dockerfile` and target `graphd`
-- [vesoft/nebula-metad](https://hub.docker.com/r/vesoft/nebula-metad): nebula-metad service built with `Dockerfile` and target `metad`
-- [vesoft/nebula-storaged](https://hub.docker.com/r/vesoft/nebula-storaged): nebula-storaged service built with `Dockerfile` and target `storaged`
-- [vesoft/nebula-tools](https://hub.docker.com/r/vesoft/nebula-tools): nebula tools built with `Dockerfile` and target `tools`, including db_dump, meta_dump and db_upgrader
+- [vesoft/nebula-graphd](https://hub.docker.com/r/vesoft/nebula-graphd): nebula-graphd service built with `Dockerfile.graphd`
+- [vesoft/nebula-metad](https://hub.docker.com/r/vesoft/nebula-metad): nebula-metad service built with `Dockerfile.metad`
+- [vesoft/nebula-storaged](https://hub.docker.com/r/vesoft/nebula-storaged): nebula-storaged service built with `Dockerfile.storaged`
+- [vesoft/nebula-tools](https://hub.docker.com/r/vesoft/nebula-tools): nebula tools built with `Dockerfile.tools`, including db_dump, meta_dump and db_upgrader

src/clients/meta/MetaClient.cpp

@@ -2484,11 +2484,6 @@ folly::Future<StatusOr<bool>> MetaClient::heartbeat() {
  }
  }
 
- // TTL for clientAddrMap
- // If multiple connections are created but do not authenticate, the clientAddrMap_ will keep
- // growing. This is to clear the clientAddrMap_ regularly.
- clearClientAddrMap();
-
  // info used in the agent, only set once
  // TOOD(spw): if we could add data path(disk) dynamicly in the future, it should be
  // reported every time it changes
@@ -3658,20 +3653,5 @@ Status MetaClient::verifyVersion() {
  return Status::OK();
 }
 
-void MetaClient::clearClientAddrMap() {
- if (clientAddrMap_.size() == 0) {
- return;
- }
-
- auto curTimestamp = time::WallClock::fastNowInSec();
- for (auto it = clientAddrMap_.cbegin(); it != clientAddrMap_.cend();) {
- // The clientAddr is expired
- if (it->second < curTimestamp) {
- it = clientAddrMap_.erase(it);
- } else {
- ++it;
- }
- }
-}
 } // namespace meta
 } // namespace nebula

src/clients/meta/MetaClient.h

@@ -155,7 +155,6 @@ using FTIndexMap = std::unordered_map<std::string, cpp2::FTIndex>;
 
 using SessionMap = std::unordered_map<SessionID, cpp2::Session>;
 
-using clientAddrMap = folly::ConcurrentHashMap<HostAddr, int64_t>;
 class MetaChangedListener {
  public:
  virtual ~MetaChangedListener() = default;
@@ -657,10 +656,6 @@ class MetaClient : public BaseMetaClient {
  return options_.localHost_.toString();
  }
 
- clientAddrMap& getClientAddrMap() {
- return clientAddrMap_;
- }
-
  protected:
  // Return true if load succeeded.
  bool loadData();
@@ -747,9 +742,6 @@ class MetaClient : public BaseMetaClient {
 
  Status verifyVersion();
 
- // Removes expired keys in the clientAddrMap_
- void clearClientAddrMap();
-
  private:
  std::shared_ptr<folly::IOThreadPoolExecutor> ioThreadPool_;
  std::shared_ptr<thrift::ThriftClientManager<cpp2::MetaServiceAsyncClient>> clientsMan_;
@@ -830,18 +822,6 @@ class MetaClient : public BaseMetaClient {
  NameIndexMap tagNameIndexMap_;
  NameIndexMap edgeNameIndexMap_;
 
- // TODO(Aiee) This is a walkaround to address the problem that using a lower version(< v2.6.0)
- // client to connect with higher version(>= v3.0.0) Nebula service will cause a crash.
- //
- // The key here is the host of the client that sends the request, and the value indicates the
- // expiration of the key because we don't want to keep the key forever.
- //
- // The assumption here is that there is ONLY ONE VERSION of the client in the host.
- //
- // This map will be updated when verifyVersion() is called. Only the clients since v2.6.0 will
- // call verifyVersion(), thus we could determine whether the client version is lower than v2.6.0
- clientAddrMap clientAddrMap_;
-
  // Global service client
  ServiceClientsList serviceClientList_;
  FTIndexMap fulltextIndexMap_;