Fix CI build (#1423)

* project restructure storage Signed-off-by: iceseer <iceseer@gmail.com> * project restructure network Signed-off-by: iceseer <iceseer@gmail.com> * project restructure block_builder Signed-off-by: iceseer <iceseer@gmail.com> * project restructure blockchain Signed-off-by: iceseer <iceseer@gmail.com> * project restructure consensus Signed-off-by: iceseer <iceseer@gmail.com> * mold Signed-off-by: turuslan <turuslan.devbox@gmail.com> * merge storage explorer and db editor into kagome Signed-off-by: turuslan <turuslan.devbox@gmail.com> * check disk usage Signed-off-by: iceseer <iceseer@gmail.com> * clear objs option Signed-off-by: iceseer <iceseer@gmail.com> * Update cmake/kagomeConfig.cmake.in Co-authored-by: Harrm <abrehchs@gmail.com> * Update core/api/CMakeLists.txt Co-authored-by: Harrm <abrehchs@gmail.com> Signed-off-by: iceseer <iceseer@gmail.com> Signed-off-by: turuslan <turuslan.devbox@gmail.com> Co-authored-by: turuslan <turuslan.devbox@gmail.com> Co-authored-by: Harrm <abrehchs@gmail.com>
qdrvm · Nov 25, 2022 · 2b6d360 · 2b6d360
1 parent bbe32ed
commit 2b6d360
Show file tree

Hide file tree

Showing 92 changed files with 393 additions and 1,237 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -36,25 +36,25 @@ jobs:
     - name: build
       env:
         DEVELOPER_DIR: /Applications/Xcode_11.7.app/Contents/Developer
-      run: ./housekeeping/make_build.sh -DCOVERAGE=OFF -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/cxx17.cmake
+      run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCOVERAGE=OFF -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/cxx17.cmake
 
   Linux:
     strategy:
       fail-fast: false
       matrix:
         options:
           - name: "Linux: gcc-10 ASAN No Toolchain"
-            run: ./housekeeping/make_build.sh -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-10_cxx17.cmake -DASAN=ON
+            run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-10_cxx17.cmake -DASAN=ON
           - name: "Linux: gcc-12 ASAN No Toolchain"
-            run: ./housekeeping/make_build.sh -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-12_cxx17.cmake -DASAN=ON
+            run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-12_cxx17.cmake -DASAN=ON
           - name: "Linux: clang-11 TSAN"
-            run: ./housekeeping/make_build.sh -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-11_cxx17.cmake -DTSAN=ON
+            run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-11_cxx17.cmake -DTSAN=ON
           - name: "Linux: clang-11 UBSAN"
-            run: ./housekeeping/make_build.sh -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-11_cxx17.cmake -DUBSAN=ON
+            run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-11_cxx17.cmake -DUBSAN=ON
           - name: "Linux: clang-11 External Project"
-            run: ./housekeeping/make_external_build.sh -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-11_cxx17.cmake
+            run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-11_cxx17.cmake
           - name: "Linux: clang-15 UBSAN"
-            run: ./housekeeping/make_build.sh -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-15_cxx17.cmake -DUBSAN=ON
+            run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-15_cxx17.cmake -DUBSAN=ON
 
     name: "${{ matrix.options.name }}"
     runs-on: ubuntu-latest
@@ -65,8 +65,19 @@ jobs:
         with:
           path: ${{ env.CACHE_PATHS }}
           key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }}
+      - name: df du
+        run: |
+          df -m || true
+          du -hd1 /__w /github || true
+      - name: Install mold
+        run: ./housekeeping/ci_install_mold.sh --make-default
       - name: "${{ matrix.options.name }}"
         run: "${{ matrix.options.run }}"
+      - name: df du
+        if: ${{ always() }}
+        run: |
+          df -m || true
+          du -hd1 /__w /github || true
 
   clang-tidy:
     name: "Linux: clang-tidy"
@@ -98,10 +109,21 @@ jobs:
         with:
           path: ${{ env.CACHE_PATHS }}
           key: ${{ github.job }}-${{ env.CACHE_VERSION }}
+      - name: df du
+        run: |
+          df -m || true
+          du -hd1 /__w /github || true
+      - name: Install mold
+        run: ./housekeeping/ci_install_mold.sh --make-default
       - name: makeBuild
         env:
           BUILD_FINAL_TARGET: ctest_coverage
         run: ./housekeeping/make_build.sh -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-10_cxx17.cmake -DCOVERAGE=ON
+      - name: df du
+        if: ${{ always() }}
+        run: |
+          df -m || true
+          du -hd1 /__w /github || true
       - if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
         name: Submit Coverage
         env:

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -40,6 +40,7 @@ option(COVERAGE     "Enable generation of coverage info"          OFF)
 option(EMBEDDINGS   "Embed developers assets"                     ON )
 option(PROFILING    "Enable internal profiling instruments"       OFF)
 option(BACKWARD     "Enable stacktrace logging instruments"       ON )
+option(CLEAR_OBJS   "Clear object files"                          OFF)
 
 # sanitizers will be enabled only for Kagome, and will be disabled for dependencies
 option(ASAN         "Enable address sanitizer"                    OFF)

diff --git a/Makefile b/Makefile
@@ -1,8 +1,7 @@
 build:
 	mkdir build && cd build && \
 		cmake -DCMAKE_BUILD_TYPE=Release .. && \
-		make kagome -j $(shell expr $(shell nproc 2>/dev/null || sysctl -n hw.ncpu) / 2 + 1 ) && \
-		make kagome-db-editor -j $(shell expr $(shell nproc 2>/dev/null || sysctl -n hw.ncpu) / 2 + 1 )
+		make kagome -j $(shell expr $(shell nproc 2>/dev/null || sysctl -n hw.ncpu) / 2 + 1 )
 
 docker: 
 	INDOCKER_IMAGE=soramitsu/kagome-dev:4-minideb BUILD_DIR=build BUILD_THREADS=$(shell expr $(shell nproc 2>/dev/null || sysctl -n hw.ncpu) + 1 ) ./housekeeping/indocker.sh ./housekeeping/make_build.sh

diff --git a/cmake/install.cmake b/cmake/install.cmake
@@ -10,6 +10,18 @@ function(kagome_install target)
         )
 endfunction()
 
+### kagome_clear_objects should be called right after target_link_libraries(target) or kagome_install(target)
+function(kagome_clear_objects target)
+    if(CLEAR_OBJS)
+        add_custom_command(TARGET ${target}
+            POST_BUILD
+            COMMAND find ${CMAKE_CURRENT_BINARY_DIR} -name "*.o" -type f -delete
+            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+            COMMENT "Removing object files from '${CMAKE_CURRENT_BINARY_DIR}'"
+        )
+    endif()
+endfunction()
+
 ### workaround for imported libraries
 function(kagome_install_mini target)
     install(TARGETS ${target} EXPORT kagomeTargets

diff --git a/cmake/kagomeConfig.cmake.in b/cmake/kagomeConfig.cmake.in
@@ -3,7 +3,9 @@
 include(CMakeFindDependencyMacro)
 
 find_dependency(Threads REQUIRED)
-find_dependency(Boost REQUIRED random filesystem)
+find_dependency(RapidJSON REQUIRED)
+find_dependency(prometheus-cpp REQUIRED)
+find_dependency(Boost REQUIRED random filesystem date_time)
 find_dependency(soralog REQUIRED)
 find_dependency(libp2p REQUIRED)
 find_dependency(xxhash REQUIRED)

diff --git a/core/api/CMakeLists.txt b/core/api/CMakeLists.txt
@@ -3,6 +3,73 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 
-add_subdirectory(jrpc)
-add_subdirectory(service)
-add_subdirectory(transport)
+add_library(api
+    service/impl/api_service_impl.cpp
+    service/chain/impl/chain_api_impl.cpp
+    service/chain/requests/get_block_hash.cpp
+    service/chain/chain_jrpc_processor.cpp
+    service/child_state/child_state_jrpc_processor.cpp
+    service/child_state/impl/child_state_api_impl.cpp
+    service/child_state/requests/get_storage_hash.cpp
+    service/child_state/requests/get_storage_size.cpp
+    service/child_state/requests/get_keys.cpp
+    service/child_state/requests/get_keys_paged.cpp
+    service/child_state/requests/get_storage.cpp
+    service/rpc/impl/rpc_api_impl.cpp
+    service/rpc/requests/methods.cpp
+    service/rpc/rpc_jrpc_processor.cpp
+    service/state/impl/state_api_impl.cpp
+    service/state/requests/call.cpp
+    service/state/requests/get_runtime_version.cpp
+    service/state/requests/unsubscribe_storage.cpp
+    service/state/requests/get_keys_paged.cpp
+    service/state/requests/get_storage.cpp
+    service/state/requests/subscribe_storage.cpp
+    service/state/state_jrpc_processor.cpp
+    service/internal/impl/internal_api_impl.cpp
+    service/internal/internal_jrpc_processor.cpp
+    service/author/impl/author_api_impl.cpp
+    service/author/author_jrpc_processor.cpp
+    service/system/impl/system_api_impl.cpp
+    service/system/requests/health.cpp
+    service/system/requests/version.cpp
+    service/system/requests/properties.cpp
+    service/system/requests/chain.cpp
+    service/system/requests/chain_type.cpp
+    service/system/requests/name.cpp
+    service/system/system_jrpc_processor.cpp
+    service/payment/impl/payment_api_impl.cpp
+    service/payment/payment_jrpc_processor.cpp
+    transport/impl/http/http_listener_impl.cpp
+    transport/impl/http/http_session.cpp
+    transport/impl/ws/ws_session.cpp
+    transport/impl/ws/ws_listener_impl.cpp
+    transport/tuner.cpp
+    transport/rpc_thread_pool.cpp
+    transport/error.cpp
+    jrpc/jrpc_handle_batch.cpp
+    jrpc/jrpc_server_impl.cpp
+    )
+
+target_link_libraries(api
+    PUBLIC
+    Boost::boost
+    logger
+    app_state_manager
+    p2p::p2p_peer_id
+    crypto_store
+    hexutil
+    scale::scale
+    storage
+    blob
+    metadata_api
+    consensus
+    ss58_codec
+    outcome
+    build_version
+    PRIVATE 
+    RapidJSON::rapidjson
+    metrics
+    )
+kagome_install(api)
+kagome_clear_objects(api)
diff --git a/core/api/jrpc/CMakeLists.txt b/core/api/jrpc/CMakeLists.txt
diff --git a/core/api/service/CMakeLists.txt b/core/api/service/CMakeLists.txt
@@ -10,7 +10,7 @@ target_link_libraries(api_service
     Boost::boost
     logger
     app_state_manager
-    rpc_thread_pool
+    api
     p2p::p2p_peer_id
     )
 

diff --git a/core/api/service/author/CMakeLists.txt b/core/api/service/author/CMakeLists.txt
diff --git a/core/api/service/chain/CMakeLists.txt b/core/api/service/chain/CMakeLists.txt
diff --git a/core/api/service/chain/requests/CMakeLists.txt b/core/api/service/chain/requests/CMakeLists.txt
diff --git a/core/api/service/child_state/CMakeLists.txt b/core/api/service/child_state/CMakeLists.txt
diff --git a/core/api/service/child_state/requests/CMakeLists.txt b/core/api/service/child_state/requests/CMakeLists.txt
diff --git a/core/api/service/internal/CMakeLists.txt b/core/api/service/internal/CMakeLists.txt
diff --git a/core/api/service/payment/CMakeLists.txt b/core/api/service/payment/CMakeLists.txt
diff --git a/core/api/service/rpc/CMakeLists.txt b/core/api/service/rpc/CMakeLists.txt
diff --git a/core/api/service/rpc/requests/CMakeLists.txt b/core/api/service/rpc/requests/CMakeLists.txt
diff --git a/core/api/service/state/CMakeLists.txt b/core/api/service/state/CMakeLists.txt