feat: disable submodule (#297)

* feat: disable submodule * fix: remove submodule from workflows * fix: get library path * fix: cmake lib path * fix: workflow for taosdump * feat: de-submodule, add .gitmodules temporary
taosdata · Jul 27, 2022 · 74b04ed · 74b04ed
1 parent 817cb6a
commit 74b04ed
Show file tree

Hide file tree

Showing 13 changed files with 85 additions and 43 deletions.
diff --git a/.github/workflows/ci-taosBenchmark-debug.yml b/.github/workflows/ci-taosBenchmark-debug.yml
@@ -88,7 +88,6 @@ jobs:
           cd TDengine/src/kit/taos-tools
           git fetch origin +refs/pull/${{env.PR_NUMBER}}/merge
           git checkout -qf FETCH_HEAD
-          git submodule update --init --recursive
 
       - name: Set up Go
         if: steps.changed-files-specific.outputs.any_changed == 'true'

diff --git a/.github/workflows/ci-taosBenchmark-release.yml b/.github/workflows/ci-taosBenchmark-release.yml
@@ -84,7 +84,6 @@ jobs:
           cd TDengine/src/kit/taos-tools
           git fetch origin +refs/pull/${{env.PR_NUMBER}}/merge
           git checkout -qf FETCH_HEAD
-          git submodule update --init --recursive
 
       - name: Set up Go
         if: steps.changed-files-specific.outputs.any_changed == 'true'

diff --git a/.github/workflows/ci-taosbenchmark-release-ws.yml b/.github/workflows/ci-taosbenchmark-release-ws.yml
@@ -82,7 +82,6 @@ jobs:
           cd TDengine/src/kit/taos-tools
           git fetch origin +refs/pull/${{env.PR_NUMBER}}/merge
           git checkout -qf FETCH_HEAD
-          git submodule update --init --recursive
 
       - name: Set up Go
         if: steps.changed-files-specific.outputs.any_changed == 'true'

diff --git a/.github/workflows/ci-taosdump-debug.yml b/.github/workflows/ci-taosdump-debug.yml
@@ -87,7 +87,6 @@ jobs:
           cd TDengine/src/kit/taos-tools
           git fetch origin +refs/pull/${{env.PR_NUMBER}}/merge
           git checkout -qf FETCH_HEAD
-          git submodule update --init --recursive
 
       - name: Set up Go
         if: steps.changed-files-specific.outputs.any_changed == 'true'
@@ -102,17 +101,17 @@ jobs:
           cd TDengine && mkdir debug && cd debug && cmake .. -DBUILD_JDBC=false -DBUILD_TOOLS=true -DTOOLS_BUILD_TYPE=Debug -DTOOLS_SANITIZE=true -DBUILD_HTTP=false && make -j8 && sudo make install
 
           pip3 install taospy numpy fabric2 psutil pandas faker
-          sudo pkill -9 taosadapter || :
-          sudo pkill -9 taosd || :
 
       - name: taosdump Func Test
         if: steps.changed-files-specific.outputs.any_changed == 'true'
         run: |
-          cd TDengine/tests/pytest
-          python3 ./test.py -f tools/taosdumpTest.py > /dev/null 2>&1 && echo -e "\033[32m $i success! \033[0m"|| echo -e "\033[31m $i failed! \033[0m" | tee -a ~/taosdump-failed.txt
-          python3 ./test.py -f tools/taosdumpTest2.py > /dev/null 2>&1 && echo -e "\033[32m $i success! \033[0m"|| echo -e "\033[31m $i failed! \033[0m" | tee -a ~/taosdump-failed.txt
-          python3 ./test.py -f tools/taosdumpTest3.py > /dev/null 2>&1 && echo -e "\033[32m $i success! \033[0m"|| echo -e "\033[31m $i failed! \033[0m" | tee -a ~/taosdump-failed.txt
-          python3 ./test.py -f tools/taosdumpTestNanoSupport.py > /dev/null 2>&1 && echo -e "\033[32m $i success! \033[0m"|| echo -e "\033[31m $i failed! \033[0m" | tee -a ~/taosdump-failed.txt
+          cd TDengine/tests/pytest || echo -e "failed to cd TDEngine/tests/pytest"
+          sudo pkill -9 taosadapter || :
+          sudo pkill -9 taosd || :
+          if find tools -name "taosdump*.py"|grep -q .;
+          then
+            for i in `find tools -name "taosdump*.py"`; do python3 ./test.py -f $i > /dev/null 2>&1 && echo -e "\033[32m $i success! \033[0m" || echo -e "\033[31m $i failed! \033[0m" | tee -a ~/taosdump-failed.txt; done
+          fi
 
       - name: taosdump System Test
         if: steps.changed-files-specific.outputs.any_changed == 'true'
@@ -122,7 +121,7 @@ jobs:
           sudo pkill -9 taosd || :
           if find 5-taos-tools/taosdump -name "*.py"|grep -q .;
           then
-            for i in `find 5-taos-tools/taosdump -name "*.py"`; do python3 ./test.py -f $i > /dev/null 2>&1 && echo -e "\033[32m system-test/$i success! \033[0m"|| echo -e "\033[31m system-test/$i failed! \033[0m" | tee -a ~/taosdump-failed.txt; done
+            for i in `find 5-taos-tools/taosdump -name "*.py"`; do python3 ./test.py -f $i > /dev/null 2>&1 && echo -e "\033[32m system-test/$i success! \033[0m" || echo -e "\033[31m system-test/$i failed! \033[0m" | tee -a ~/taosdump-failed.txt; done
           fi
 
       - name: taosdump Old Test

diff --git a/.github/workflows/ci-taosdump-release-ws.yml b/.github/workflows/ci-taosdump-release-ws.yml
@@ -84,7 +84,6 @@ jobs:
           cd TDengine/src/kit/taos-tools
           git fetch origin +refs/pull/${{env.PR_NUMBER}}/merge
           git checkout -qf FETCH_HEAD
-          git submodule update --init --recursive
 
       - name: Set up Go
         if: steps.changed-files-specific.outputs.any_changed == 'true'

diff --git a/.github/workflows/ci-taosdump-release.yml b/.github/workflows/ci-taosdump-release.yml
@@ -84,7 +84,6 @@ jobs:
           cd TDengine/src/kit/taos-tools
           git fetch origin +refs/pull/${{env.PR_NUMBER}}/merge
           git checkout -qf FETCH_HEAD
-          git submodule update --init --recursive
 
       - name: Set up Go
         if: steps.changed-files-specific.outputs.any_changed == 'true'
@@ -106,10 +105,10 @@ jobs:
           sudo pkill -9 taosadapter || :
           sudo pkill -9 taosd || :
 
-          python3 ./test.py -f tools/taosdumpTest.py > /dev/null 2>&1 && echo -e "\033[32m $i success! \033[0m"|| echo -e "\033[31m $i failed! \033[0m" | tee -a ~/taosdump-failed.txt
-          python3 ./test.py -f tools/taosdumpTest2.py > /dev/null 2>&1 && echo -e "\033[32m $i success! \033[0m"|| echo -e "\033[31m $i failed! \033[0m" | tee -a ~/taosdump-failed.txt
-          python3 ./test.py -f tools/taosdumpTest3.py > /dev/null 2>&1 && echo -e "\033[32m $i success! \033[0m"|| echo -e "\033[31m $i failed! \033[0m" | tee -a ~/taosdump-failed.txt
-          python3 ./test.py -f tools/taosdumpTestNanoSupport.py > /dev/null 2>&1 && echo -e "\033[32m $i success! \033[0m"|| echo -e "\033[31m $i failed! \033[0m" | tee -a ~/taosdump-failed.txt
+          if find tools -name "taosdump*.py"|grep -q .;
+          then
+            for i in `find tools -name "taosdump*.py"`; do python3 ./test.py -f $i > /dev/null 2>&1 && echo -e "\033[32m $i success! \033[0m" || echo -e "\033[31m $i failed! \033[0m" | tee -a ~/taosdump-failed.txt; done
+          fi
 
       - name: taosdump System Test
         if: steps.changed-files-specific.outputs.any_changed == 'true'

diff --git a/.github/workflows/coveralls.yml b/.github/workflows/coveralls.yml
@@ -105,7 +105,6 @@ jobs:
           cd TDengine/src/kit/taos-tools
           git fetch origin +refs/pull/${{env.PR_NUMBER}}/merge
           git checkout -qf FETCH_HEAD
-          git submodule update --init --recursive
 
       - name: Set up Go
         if: steps.changed-files-specific.outputs.any_changed == 'true'

diff --git a/.github/workflows/macos-release.yml b/.github/workflows/macos-release.yml
@@ -75,7 +75,6 @@ jobs:
           cd TDengine/src/kit/taos-tools
           git fetch origin +refs/pull/${{env.PR_NUMBER}}/merge
           git checkout -qf FETCH_HEAD
-          git submodule update --init --recursive
 
       - name: Set up homebrew
         if: steps.changed-files-specific.outputs.any_changed == 'true'

diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
@@ -75,7 +75,6 @@ jobs:
           cd TDengine/src/kit/taos-tools
           git fetch origin +refs/pull/${{env.PR_NUMBER}}/merge
           git checkout -qf FETCH_HEAD
-          git submodule update --init --recursive
 
       - name: Set up homebrew
         if: steps.changed-files-specific.outputs.any_changed == 'true'

diff --git a/.github/workflows/non-x64.yml b/.github/workflows/non-x64.yml
@@ -40,7 +40,6 @@ jobs:
           arch: ${{ matrix.arch }}
           distro: ${{ matrix.distro }}
           ref: develop
-          submodules: recursive
           path: 'taos-tools'
           # Set an output parameter `uname` for use in subsequent steps
           run: |
@@ -114,12 +113,18 @@ jobs:
         if: |
           github.event_name == 'pull_request' &&
           steps.changed-files-specific.outputs.any_changed == 'true'
-        run: |
-          echo "The uname output was ${{ steps.setup.outputs.uname }}"
-          cd TDengine/src/kit/taos-tools
-          git fetch origin +refs/pull/${{env.PR_NUMBER}}/merge
-          git checkout -qf FETCH_HEAD
-          git submodule update --init --recursive
+        uses: uraimo/run-on-arch-action@v2.1.1
+        with:
+          arch: ${{ matrix.arch }}
+          distro: ${{ matrix.distro }}
+          run: |
+            echo "The uname output was ${{ steps.setup.outputs.uname }}"
+            apt update -y > /dev/null 2>&1
+            apt install git cmake build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev pkg-config -y > /dev/null 2>&1
+            git config --global --add safe.directory /home/runner/work/taos-tools/taos-tools/TDengine/src/kit/taos-tools
+            cd TDengine/src/kit/taos-tools
+            git fetch origin +refs/pull/${{env.PR_NUMBER}}/merge
+            git checkout -qf FETCH_HEAD
 
       - name: Build tdengine
         if: steps.changed-files-specific.outputs.any_changed == 'true'
@@ -129,8 +134,8 @@ jobs:
           distro: ${{ matrix.distro }}
           run: |
             echo "The uname output was ${{ steps.setup.outputs.uname }}"
-            apt update -y
-            apt install cmake build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev pkg-config -y
+            apt update -y > /dev/null 2>&1
+            apt install git cmake build-essential libjansson-dev libsnappy-dev liblzma-dev libz-dev pkg-config -y > /dev/null 2>&1
             cd TDengine
             mkdir debug
             cd debug

diff --git a/.gitignore b/.gitignore
@@ -11,6 +11,9 @@ dump_result.txt
 sim
 tests/taosdump/native/*.sql
 tests/taosdump/ws/*.sql
+deps/avro
+deps/jansson
+deps/snappy
 
 # Prerequisites
 *.d

diff --git a/deps/CMakeLists.txt b/deps/CMakeLists.txt
@@ -1,11 +1,33 @@
+
+INCLUDE(ExternalProject)
+
+OPTION(JANSSON_BUILD_DOCS "turn jansson build docs off" OFF)
+OPTION(JANSSON_INSTALL "turn jansson install off" OFF)
+
+ExternalProject_Add(
+    deps-jansson
+    GIT_REPOSITORY https://github.com/akheron/jansson
+    GIT_TAG         v2.14
+    GIT_SHALLOW     true
+    PREFIX ${CMAKE_CURRENT_BINARY_DIR}/jansson
+    SOURCE_DIR ${PROJECT_SOURCE_DIR}/deps/jansson
+    CONFIGURE_COMMAND cmake -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/build/ -S ${PROJECT_SOURCE_DIR}/deps/jansson
+    )
+
+ExternalProject_Add(
+    deps-snappy
+    GIT_REPOSITORY https://github.com/google/snappy
+    GIT_TAG         1.1.8
+    GIT_SHALLOW     true
+    PREFIX ${CMAKE_CURRENT_BINARY_DIR}/snappy
+    SOURCE_DIR ${PROJECT_SOURCE_DIR}/deps/snappy
+    CONFIGURE_COMMAND cmake -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/build/ -S ${PROJECT_SOURCE_DIR}/deps/snappy
+    )
+
 IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
     IF (${TOOLS_BUILD_TYPE} MATCHES "debug")
         MESSAGE("will not build jansson and snappy, use system's")
     ELSE ()
-        OPTION(JANSSON_BUILD_DOCS "turn jansson build docs off" OFF)
-        OPTION(JANSSON_INSTALL "turn jansson install off" OFF)
-        ADD_SUBDIRECTORY(jansson)
-
         OPTION(BUILD_SHARED_LIBS "turn snappy build shared lib off" OFF)
         OPTION(SNAPPY_BUILD_TESTS "turn snappy build tests off" OFF)
         OPTION(SNAPPY_BUILD_BENCHMARKS "turn snappy build benchmarks off" OFF)
@@ -17,25 +39,23 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
                 set (TEMP_CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG})
                 string(REPLACE "-Wno-pointer-to-int-cast" "" CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG})
                 string(REPLACE "-Wno-incompatible-pointer-types" "" CMAKE_CXX_FLAGS_DEBUG ${CMAKE_CXX_FLAGS_DEBUG})
-                ADD_SUBDIRECTORY(snappy)
                 set(CMAKE_CXX_FLAGS_DEBUG ${TEMP_CMAKE_CXX_FLAGS_DEBUG})
             ELSE ()
                 set (TEMP_CMAKE_CXX_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE})
                 string(REPLACE "-Wno-pointer-to-int-cast" "" CMAKE_CXX_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE})
                 string(REPLACE "-Wno-incompatible-pointer-types" "" CMAKE_CXX_FLAGS_RELEASE ${CMAKE_CXX_FLAGS_RELEASE})
-                ADD_SUBDIRECTORY(snappy)
                 set(CMAKE_CXX_FLAGS_RELEASE ${TEMP_CMAKE_CXX_FLAGS_RELEASE})
             ENDIF ()
-        ELSE()
-            ADD_SUBDIRECTORY(snappy)
         ENDIF()
     ENDIF ()
 
-    INCLUDE(ExternalProject)
     ExternalProject_Add(
         apache-avro
+        GIT_REPOSITORY https://github.com/apache/avro
+        GIT_TAG         release-1.11.0
+        GIT_SHALLOW     true
         PREFIX ${CMAKE_CURRENT_BINARY_DIR}/avro
-        SOURCE_DIR ${PROJECT_SOURCE_DIR}/deps/avro/lang/c
+        SOURCE_DIR ${PROJECT_SOURCE_DIR}/deps/avro
         CONFIGURE_COMMAND cmake -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}/build/ -S ${PROJECT_SOURCE_DIR}/deps/avro/lang/c
     )
     ADD_SUBDIRECTORY(CUnit)

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -221,10 +221,33 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ${CMAKE_SYSTEM_NAME} MATCHES "Darwin
             TARGET_LINK_LIBRARIES(taosBenchmark taos pthread toolscJson argp ${OS_LINK_FLAGS} ${WEBSOCKET_LINK_FLAGS})
         ELSE ()
             ADD_LIBRARY(avro STATIC IMPORTED)
-            IF(${OS_ID} MATCHES "centos" OR ${OS_ID} MATCHES "kylin" OR ${OS_ID} MATCHES "rhel" OR ${OS_ID} MATCHES "rocky")
-                SET_PROPERTY(TARGET avro PROPERTY IMPORTED_LOCATION "${CMAKE_BINARY_DIR}/build/lib64/libavro.a")
+            IF(${OS_ID} MATCHES "centos" OR ${OS_ID} MATCHES "kylin" 
+                    OR ${OS_ID} MATCHES "rhel" OR ${OS_ID} MATCHES "rocky")
+                SET_PROPERTY(TARGET avro PROPERTY IMPORTED_LOCATION 
+                    "${CMAKE_BINARY_DIR}/build/lib64/libavro.a")
             ELSE()
-                SET_PROPERTY(TARGET avro PROPERTY IMPORTED_LOCATION "${CMAKE_BINARY_DIR}/build/lib/libavro.a")
+                SET_PROPERTY(TARGET avro PROPERTY IMPORTED_LOCATION 
+                    "${CMAKE_BINARY_DIR}/build/lib/libavro.a")
+            ENDIF()
+
+            ADD_LIBRARY(jansson STATIC IMPORTED)
+            IF(${OS_ID} MATCHES "centos" OR ${OS_ID} MATCHES "kylin" 
+                    OR ${OS_ID} MATCHES "rhel" OR ${OS_ID} MATCHES "rocky")
+                SET_PROPERTY(TARGET jansson PROPERTY IMPORTED_LOCATION 
+                    "${CMAKE_BINARY_DIR}/build/lib64/libjansson.a")
+            ELSE()
+                SET_PROPERTY(TARGET jansson PROPERTY IMPORTED_LOCATION 
+                    "${CMAKE_BINARY_DIR}/build/lib/libjansson.a")
+            ENDIF()
+
+            ADD_LIBRARY(snappy STATIC IMPORTED)
+            IF(${OS_ID} MATCHES "centos" OR ${OS_ID} MATCHES "kylin" 
+                    OR ${OS_ID} MATCHES "rhel" OR ${OS_ID} MATCHES "rocky")
+                SET_PROPERTY(TARGET snappy PROPERTY IMPORTED_LOCATION 
+                    "${CMAKE_BINARY_DIR}/build/lib64/libsnappy.a")
+            ELSE()
+                SET_PROPERTY(TARGET snappy PROPERTY IMPORTED_LOCATION 
+                    "${CMAKE_BINARY_DIR}/build/lib/libsnappy.a")
             ENDIF()
 
             EXECUTE_PROCESS (