From 42b9c8981a08ddaf611a068baa82d583872c5315 Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Wed, 10 Jun 2020 17:15:21 +0200 Subject: [PATCH 01/39] Add build targets for running tests --- .github/workflows/centos-ci.yml | 2 +- .github/workflows/macos-ci.yml | 2 +- .github/workflows/ubuntu-ci.yml | 2 +- cmake/Testing.cmake | 14 ++++++++++++-- doc/dev_guide/build.md | 7 +++++-- 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/.github/workflows/centos-ci.yml b/.github/workflows/centos-ci.yml index 44a5bdb22..592789059 100644 --- a/.github/workflows/centos-ci.yml +++ b/.github/workflows/centos-ci.yml @@ -82,7 +82,7 @@ jobs: . bin/thisbdm.sh export DISPLAY=:99.0 ../util/xvfb-initd.sh start - bin/biodynamo-unit-tests + ninja run-unit-tests ../util/xvfb-initd.sh stop - name: Notify Slack diff --git a/.github/workflows/macos-ci.yml b/.github/workflows/macos-ci.yml index 2756603f0..8de41932d 100644 --- a/.github/workflows/macos-ci.yml +++ b/.github/workflows/macos-ci.yml @@ -61,7 +61,7 @@ jobs: working-directory: build run: | . bin/thisbdm.sh - bin/biodynamo-unit-tests + ninja run-unit-tests - name: Notify Slack uses: 8398a7/action-slack@v3 diff --git a/.github/workflows/ubuntu-ci.yml b/.github/workflows/ubuntu-ci.yml index 8ba6acc8e..edab8d268 100644 --- a/.github/workflows/ubuntu-ci.yml +++ b/.github/workflows/ubuntu-ci.yml @@ -70,7 +70,7 @@ jobs: . bin/thisbdm.sh export DISPLAY=:99.0 ../util/xvfb-initd.sh start - bin/biodynamo-unit-tests + ninja run-unit-tests ../util/xvfb-initd.sh stop - name: Notify Slack diff --git a/cmake/Testing.cmake b/cmake/Testing.cmake index 6fbbc0bfb..b2ec44ab9 100644 --- a/cmake/Testing.cmake +++ b/cmake/Testing.cmake @@ -40,7 +40,11 @@ set_target_properties(libgtest PROPERTIES include_directories("${CMAKE_BINARY_DIR}/gtest/src/gtest/include") # create target that shows the test ouput on failure -add_custom_target(check COMMAND ${CMAKE_CTEST_COMMAND} --force-new-ctest-process --output-on-failure) +add_custom_target(run-check COMMAND ${CMAKE_CTEST_COMMAND} --force-new-ctest-process --output-on-failure) + +# create target for running biodynamo-unit-tests +add_custom_target(run-unit-tests COMMAND ${CMAKE_BINARY_DIR}/bin/biodynamo-unit-tests) +add_dependencies(run-unit-tests biodynamo-unit-tests) # add custom clean target for test project add_custom_target(testbdmclean) @@ -78,9 +82,15 @@ function(bdm_add_test_executable TEST_TARGET) # filter out SchedulerTest.Backup because of timing issue add_test(NAME "valgrind_${TEST_TARGET}" COMMAND ${CMAKE_BINARY_DIR}/launcher.sh ${CMAKE_SOURCE_DIR}/util/valgrind.sh ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TEST_TARGET} -- --gtest_filter=-*DeathTest.*:IOTest.InvalidRead:SchedulerTest.Backup:ResourceManagerTest.SortAndApplyOnAllElementsParallel*:InlineVector*:NeuriteElementBehaviour.*:MechanicalInteraction.*) + add_custom_target(run-valgrind + COMMAND ${CMAKE_BINARY_DIR}/launcher.sh ${CMAKE_SOURCE_DIR}/util/valgrind.sh ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TEST_TARGET} -- --gtest_filter=-*DeathTest.*:IOTest.InvalidRead:SchedulerTest.Backup:ResourceManagerTest.SortAndApplyOnAllElementsParallel*:InlineVector*:NeuriteElementBehaviour.*:MechanicalInteraction.*) + add_dependencies(run-valgrind biodynamo-unit-tests) endif() - add_dependencies(check ${TEST_TARGET}) + add_dependencies(run-check ${TEST_TARGET}) + + # add target for system tests + add_custom_target(run-demos COMMAND ${CMAKE_SOURCE_DIR}/test/system-test.sh) add_custom_target("testbdmclean_${TEST_TARGET}" COMMAND ${CMAKE_COMMAND} -P "${CMAKE_BINARY_DIR}/CMakeFiles/${TEST_TARGET}.dir/cmake_clean.cmake") add_dependencies(testbdmclean "testbdmclean_${TEST_TARGET}") diff --git a/doc/dev_guide/build.md b/doc/dev_guide/build.md index 38033e918..ef2385e4c 100644 --- a/doc/dev_guide/build.md +++ b/doc/dev_guide/build.md @@ -137,9 +137,12 @@ If you change the value of these switches, you might have to delete `CMakeCache. | Target | Description | | --------------- | ------------ | -| `test` | executes all tests | -| `check` | executes all tests and shows test output on failure | +| `run-unit-tests` | executes all BioDynaMo unit tests | +| `run-valgrind` | executes BioDynaMo valgrind tests | +| `run-check` | executes both unit and valgrind tests | +| `run-demos` | executes all demos and integration tests | | `clean` | will clean all targets, also the external projects | +| `cleanbuild` | will clean everything in the build directory, except for third_party (useful for avoiding downloading third party software) | | `bdmclean` | will only clean the `biodynamo` and `runBiodynamoTests*` targets | | `testbdmclean` | will only clean the `runBiodynamoTests*` target | | `doc` | will generate the API, user and developer documentation in directory `build/doc`. | From f0c5b28a803d478b1687657a4541a2d2f8b8a143 Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Wed, 10 Jun 2020 17:16:04 +0200 Subject: [PATCH 02/39] Check for ParaView / SBML support for sytem tests --- test/system/multiple_simulations.sh | 9 +++++++++ test/system/sbml_integration.sh | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/test/system/multiple_simulations.sh b/test/system/multiple_simulations.sh index 124407019..aa5fea671 100755 --- a/test/system/multiple_simulations.sh +++ b/test/system/multiple_simulations.sh @@ -22,6 +22,15 @@ biodynamo demo multiple_simulations "${tmp_dir}" cd "${tmp_dir}/multiple_simulations" biodynamo run +# Check if ParaView was enabled for this BioDynaMo installation +set +e +bdm-config --config | grep -i paraview +rc_pv=$? +set -e +if [ $rc_pv -ne 0 ]; then + exit 0 +fi + # check if ParaView files exists # first simulation [ -f output/multiple_simulations/simulation_info.json ] diff --git a/test/system/sbml_integration.sh b/test/system/sbml_integration.sh index 90f2b07bb..d429526f2 100755 --- a/test/system/sbml_integration.sh +++ b/test/system/sbml_integration.sh @@ -26,6 +26,13 @@ if [ "$BDM_OS" = "osx" ]; then exit 0 fi +# Check if SBML was enabled for this BioDynaMo installation +set +e +bdm-config --config | grep -i sbml +rc_sbml=$? +set -e +if [ $rc_sbml -ne 0 ]; then + demo_name="sbml_integration" demo_dir=$(mktemp -d) biodynamo demo "${demo_name}" "${demo_dir}" From 5aaf4abf2e2859e54db2b7a5cf7e8dd1693aa5bb Mon Sep 17 00:00:00 2001 From: Fons Rademakers Date: Thu, 11 Jun 2020 04:00:03 +0200 Subject: [PATCH 03/39] Generalize warning message. --- src/core/util/thread_info.h | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/core/util/thread_info.h b/src/core/util/thread_info.h index fab74911d..d28308acf 100644 --- a/src/core/util/thread_info.h +++ b/src/core/util/thread_info.h @@ -154,10 +154,9 @@ class ThreadInfo { // even though OMP_PROC_BIND is set to true. // A performance analysis showed almost identical results between true, // and spread. - Log::Warning("ThreadInfo", - "The environmental variable OMP_PROC_BIND must be set to " - "true. On Linux run 'export OMP_PROC_BIND=true' prior to " - "running BioDynaMo"); + Log::Warning("ThreadInfo::ThreadInfo", + "The environment variable OMP_PROC_BIND must be set to " + "true prior to running BioDynaMo ('export OMP_PROC_BIND=true')"); } Renew(); } From 05db87da65b92b90a50c573eb9559888d6f34641 Mon Sep 17 00:00:00 2001 From: Fons Rademakers Date: Thu, 11 Jun 2020 04:00:29 +0200 Subject: [PATCH 04/39] Fix cmake RPATH settings for macOS and introduce -Drpath option for Linux. On macOS the rpath is now correctly to be relative to the executable. The whole tree (bin, lib, etc) is relocatable and there is no need to set DYLD_LIBRARY_PATH anymore. With the new option -Drpath=ON also Linux executables will get the rpath set, however the code is not relocatable as the rpath is absolute, however there is no need to set LD_LIBRARY_PATH anymore. --- CMakeLists.txt | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2c9d63a67..a72c9203c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -113,6 +113,7 @@ option(tcmalloc "Use tcmalloc for memory allocations." OFF) option(jemalloc "Use jemalloc for memory allocations." OFF) option(website "Enable website generation (make website<-live>)." OFF) option(valgrind "Enable valgrind tests and make build compatible with valgrind tool." ON) +option(rpath "Link libraries with built-in RPATH (run-time search path)." OFF) if(APPLE) find_program(BREW_BIN brew) @@ -394,16 +395,22 @@ endif() include(SetCompilerFlags) # -------------------- set rpath options --------------------------------------- -# When building, use the RPATH -set(CMAKE_INSTALL_RPATH "") -set(CMAKE_BUILD_RPATH "") -set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) -set(CMAKE_SKIP_BUILD_RPATH TRUE) # skip the full RPATH for the build tree -set(CMAKE_INSTALL_RPATH_USE_LINK_PATH FALSE) -set(CMAKE_MACOSX_RPATH TRUE) # use @rpath keyword for the MacOS rpath -# Sets @rpath for libbiodynamo.dylib on Mac OS in the build directory -if (APPLE) - SET(CMAKE_BUILD_WITH_INSTALL_RPATH ON) +# When building, don't use the install RPATH already (but later on when installing) +set(CMAKE_SKIP_BUILD_RPATH FALSE) # don't skip the full RPATH for the build tree +set(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) # use always the build RPATH for the build tree +set(CMAKE_MACOSX_RPATH TRUE) # use RPATH for MacOSX +set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) # point to directories outside the build tree to the install RPATH + +# Check whether to add RPATH to the installation (the build tree always has the RPATH enabled) +if(rpath) + set(CMAKE_INSTALL_RPATH ${CMAKE_INSTALL_FULL_LIBDIR}) # install LIBDIR + set(CMAKE_SKIP_INSTALL_RPATH FALSE) # don't skip the full RPATH for the install tree +elseif(APPLE) + set(CMAKE_INSTALL_NAME_DIR "@rpath") + set(CMAKE_INSTALL_RPATH "@loader_path/../lib") # self relative LIBDIR + set(CMAKE_SKIP_INSTALL_RPATH FALSE) # don't skip the full RPATH for the install tree +else() + set(CMAKE_SKIP_INSTALL_RPATH TRUE) # skip the full RPATH for the install tree endif() # -------------------- extract third party libraries --------------------------- From cb173d612ef5b2d7b3aa186414d637781ab7be03 Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Wed, 10 Jun 2020 17:30:50 +0200 Subject: [PATCH 05/39] Add cron jobs for system tests in GH Actions --- .github/workflows/centos-ci.yml | 13 +++++++++++++ .github/workflows/macos-ci.yml | 13 +++++++++++++ .github/workflows/ubuntu-ci.yml | 13 +++++++++++++ 3 files changed, 39 insertions(+) diff --git a/.github/workflows/centos-ci.yml b/.github/workflows/centos-ci.yml index 592789059..42fc0b577 100644 --- a/.github/workflows/centos-ci.yml +++ b/.github/workflows/centos-ci.yml @@ -85,6 +85,19 @@ jobs: ninja run-unit-tests ../util/xvfb-initd.sh stop + - name: System tests BioDynaMo + shell: bash + working-directory: build + on: + schedule: + - cron: "3 * * *" + run: | + . bin/thisbdm.sh + export DISPLAY=:99.0 + ../util/xvfb-initd.sh start + ninja run-demos + ../util/xvfb-initd.sh stop + - name: Notify Slack uses: 8398a7/action-slack@v3 with: diff --git a/.github/workflows/macos-ci.yml b/.github/workflows/macos-ci.yml index 8de41932d..18d979745 100644 --- a/.github/workflows/macos-ci.yml +++ b/.github/workflows/macos-ci.yml @@ -63,6 +63,19 @@ jobs: . bin/thisbdm.sh ninja run-unit-tests + - name: System tests BioDynaMo + shell: bash + working-directory: build + on: + schedule: + - cron: "3 * * *" + run: | + . bin/thisbdm.sh + export DISPLAY=:99.0 + ../util/xvfb-initd.sh start + ninja run-demos + ../util/xvfb-initd.sh stop + - name: Notify Slack uses: 8398a7/action-slack@v3 with: diff --git a/.github/workflows/ubuntu-ci.yml b/.github/workflows/ubuntu-ci.yml index edab8d268..0bd533d43 100644 --- a/.github/workflows/ubuntu-ci.yml +++ b/.github/workflows/ubuntu-ci.yml @@ -73,6 +73,19 @@ jobs: ninja run-unit-tests ../util/xvfb-initd.sh stop + - name: System tests BioDynaMo + shell: bash + working-directory: build + on: + schedule: + - cron: "3 * * *" + run: | + . bin/thisbdm.sh + export DISPLAY=:99.0 + ../util/xvfb-initd.sh start + ninja run-demos + ../util/xvfb-initd.sh stop + - name: Notify Slack uses: 8398a7/action-slack@v3 with: From 79617e1bb5d15dbbb38528a2e89ce43ec8578991 Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Fri, 12 Jun 2020 00:17:30 +0200 Subject: [PATCH 06/39] Test cron jobs: website and demos --- .github/workflows/centos-ci.yml | 6 +++--- .github/workflows/macos-ci.yml | 6 +++--- .github/workflows/ubuntu-ci.yml | 19 ++++++++++++++++--- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/.github/workflows/centos-ci.yml b/.github/workflows/centos-ci.yml index 42fc0b577..34b9e74e5 100644 --- a/.github/workflows/centos-ci.yml +++ b/.github/workflows/centos-ci.yml @@ -88,9 +88,9 @@ jobs: - name: System tests BioDynaMo shell: bash working-directory: build - on: - schedule: - - cron: "3 * * *" + # on: + # schedule: + # - cron: "3 * * *" run: | . bin/thisbdm.sh export DISPLAY=:99.0 diff --git a/.github/workflows/macos-ci.yml b/.github/workflows/macos-ci.yml index 18d979745..baf47704f 100644 --- a/.github/workflows/macos-ci.yml +++ b/.github/workflows/macos-ci.yml @@ -66,9 +66,9 @@ jobs: - name: System tests BioDynaMo shell: bash working-directory: build - on: - schedule: - - cron: "3 * * *" + # on: + # schedule: + # - cron: "3 * * *" run: | . bin/thisbdm.sh export DISPLAY=:99.0 diff --git a/.github/workflows/ubuntu-ci.yml b/.github/workflows/ubuntu-ci.yml index 0bd533d43..1f4fc3be9 100644 --- a/.github/workflows/ubuntu-ci.yml +++ b/.github/workflows/ubuntu-ci.yml @@ -76,9 +76,22 @@ jobs: - name: System tests BioDynaMo shell: bash working-directory: build - on: - schedule: - - cron: "3 * * *" + # on: + # schedule: + # - cron: "3 * * *" + run: | + . bin/thisbdm.sh + export DISPLAY=:99.0 + ../util/xvfb-initd.sh start + ninja run-demos + ../util/xvfb-initd.sh stop + + - name: Build and deploy biodynamo.org + shell: bash + working-directory: build + # on: + # schedule: + # - cron: "3 * * *" run: | . bin/thisbdm.sh export DISPLAY=:99.0 From 6e979a478754eb786fbcc6ddd4b871a8022d7456 Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Fri, 12 Jun 2020 00:28:12 +0200 Subject: [PATCH 07/39] Add git test user for system tests --- .github/workflows/centos-ci.yml | 2 ++ .github/workflows/macos-ci.yml | 2 ++ .github/workflows/ubuntu-ci.yml | 15 ++------------- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/.github/workflows/centos-ci.yml b/.github/workflows/centos-ci.yml index 34b9e74e5..6a9870b9e 100644 --- a/.github/workflows/centos-ci.yml +++ b/.github/workflows/centos-ci.yml @@ -93,6 +93,8 @@ jobs: # - cron: "3 * * *" run: | . bin/thisbdm.sh + git config --system user.name "Test User" + git config --system user.email user@test.com export DISPLAY=:99.0 ../util/xvfb-initd.sh start ninja run-demos diff --git a/.github/workflows/macos-ci.yml b/.github/workflows/macos-ci.yml index baf47704f..74bd6d62c 100644 --- a/.github/workflows/macos-ci.yml +++ b/.github/workflows/macos-ci.yml @@ -71,6 +71,8 @@ jobs: # - cron: "3 * * *" run: | . bin/thisbdm.sh + git config --system user.name "Test User" + git config --system user.email user@test.com export DISPLAY=:99.0 ../util/xvfb-initd.sh start ninja run-demos diff --git a/.github/workflows/ubuntu-ci.yml b/.github/workflows/ubuntu-ci.yml index 1f4fc3be9..e454d75e1 100644 --- a/.github/workflows/ubuntu-ci.yml +++ b/.github/workflows/ubuntu-ci.yml @@ -68,6 +68,8 @@ jobs: working-directory: build run: | . bin/thisbdm.sh + git config --system user.name "Test User" + git config --system user.email user@test.com export DISPLAY=:99.0 ../util/xvfb-initd.sh start ninja run-unit-tests @@ -86,19 +88,6 @@ jobs: ninja run-demos ../util/xvfb-initd.sh stop - - name: Build and deploy biodynamo.org - shell: bash - working-directory: build - # on: - # schedule: - # - cron: "3 * * *" - run: | - . bin/thisbdm.sh - export DISPLAY=:99.0 - ../util/xvfb-initd.sh start - ninja run-demos - ../util/xvfb-initd.sh stop - - name: Notify Slack uses: 8398a7/action-slack@v3 with: From 13c6d691fe462a513acda8a9383130bda119460c Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Fri, 12 Jun 2020 00:39:25 +0200 Subject: [PATCH 08/39] Fix mistake of adding git user to unit tests instead of system tests --- .github/workflows/ubuntu-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ubuntu-ci.yml b/.github/workflows/ubuntu-ci.yml index e454d75e1..ca8550f12 100644 --- a/.github/workflows/ubuntu-ci.yml +++ b/.github/workflows/ubuntu-ci.yml @@ -68,8 +68,6 @@ jobs: working-directory: build run: | . bin/thisbdm.sh - git config --system user.name "Test User" - git config --system user.email user@test.com export DISPLAY=:99.0 ../util/xvfb-initd.sh start ninja run-unit-tests @@ -83,6 +81,8 @@ jobs: # - cron: "3 * * *" run: | . bin/thisbdm.sh + git config --system user.name "Test User" + git config --system user.email user@test.com export DISPLAY=:99.0 ../util/xvfb-initd.sh start ninja run-demos From bd162e3563a9929cf8dca4a4e7e02e93bdeed53b Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Fri, 12 Jun 2020 00:50:25 +0200 Subject: [PATCH 09/39] Try sudo --- .github/workflows/centos-ci.yml | 4 ++-- .github/workflows/ubuntu-ci.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/centos-ci.yml b/.github/workflows/centos-ci.yml index 6a9870b9e..6bdfa9a9d 100644 --- a/.github/workflows/centos-ci.yml +++ b/.github/workflows/centos-ci.yml @@ -93,8 +93,8 @@ jobs: # - cron: "3 * * *" run: | . bin/thisbdm.sh - git config --system user.name "Test User" - git config --system user.email user@test.com + sudo git config --system user.name "Test User" + sudo git config --system user.email user@test.com export DISPLAY=:99.0 ../util/xvfb-initd.sh start ninja run-demos diff --git a/.github/workflows/ubuntu-ci.yml b/.github/workflows/ubuntu-ci.yml index ca8550f12..511db2c94 100644 --- a/.github/workflows/ubuntu-ci.yml +++ b/.github/workflows/ubuntu-ci.yml @@ -81,8 +81,8 @@ jobs: # - cron: "3 * * *" run: | . bin/thisbdm.sh - git config --system user.name "Test User" - git config --system user.email user@test.com + sudo git config --system user.name "Test User" + sudo git config --system user.email user@test.com export DISPLAY=:99.0 ../util/xvfb-initd.sh start ninja run-demos From 2394101c4c4bb252b6d0bf5068688abf52fc4a6b Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Fri, 12 Jun 2020 00:59:50 +0200 Subject: [PATCH 10/39] Fix broken if statement --- test/system/sbml_integration.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/system/sbml_integration.sh b/test/system/sbml_integration.sh index d429526f2..08955ad21 100755 --- a/test/system/sbml_integration.sh +++ b/test/system/sbml_integration.sh @@ -32,6 +32,8 @@ bdm-config --config | grep -i sbml rc_sbml=$? set -e if [ $rc_sbml -ne 0 ]; then + exit 0 +fi demo_name="sbml_integration" demo_dir=$(mktemp -d) From e76f4ebfc16f39e173944d6db8e20f804415dec0 Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Fri, 12 Jun 2020 01:40:58 +0200 Subject: [PATCH 11/39] Enable website building and deployment --- .github/workflows/macos-ci.yml | 6 +++--- .github/workflows/ubuntu-ci.yml | 25 ++++++++++++++++++++++--- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/.github/workflows/macos-ci.yml b/.github/workflows/macos-ci.yml index 74bd6d62c..3e9fff3d1 100644 --- a/.github/workflows/macos-ci.yml +++ b/.github/workflows/macos-ci.yml @@ -66,9 +66,9 @@ jobs: - name: System tests BioDynaMo shell: bash working-directory: build - # on: - # schedule: - # - cron: "3 * * *" + on: + schedule: + - cron: "3 * * *" run: | . bin/thisbdm.sh git config --system user.name "Test User" diff --git a/.github/workflows/ubuntu-ci.yml b/.github/workflows/ubuntu-ci.yml index 511db2c94..5109663b2 100644 --- a/.github/workflows/ubuntu-ci.yml +++ b/.github/workflows/ubuntu-ci.yml @@ -76,9 +76,9 @@ jobs: - name: System tests BioDynaMo shell: bash working-directory: build - # on: - # schedule: - # - cron: "3 * * *" + on: + schedule: + - cron: "3 * * *" run: | . bin/thisbdm.sh sudo git config --system user.name "Test User" @@ -88,6 +88,25 @@ jobs: ninja run-demos ../util/xvfb-initd.sh stop + - name: Build biodynamo.org + shell: bash + working-directory: build + run: | + . bin/thisbdm.sh + make website + + - name: Deploy biodynamo.org + shell: bash + uses: peaceiris/actions-gh-pages@v3 + with: + deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }} + external_repository: BioDynaMo/biodynamo.github.io + publish_branch: master + user_name: 'github-actions[bot]' + user_email: 'bdmtravis@gmail.com' + commit_message: ${{ github.event.head_commit.message }} + publish_dir: ./website/public + - name: Notify Slack uses: 8398a7/action-slack@v3 with: From cdbe42339840d2581908a401024dda5d866faf0d Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Fri, 12 Jun 2020 01:59:18 +0200 Subject: [PATCH 12/39] Remove cron syntax temporarily --- .github/workflows/centos-ci.yml | 3 --- .github/workflows/macos-ci.yml | 3 --- .github/workflows/ubuntu-ci.yml | 3 --- 3 files changed, 9 deletions(-) diff --git a/.github/workflows/centos-ci.yml b/.github/workflows/centos-ci.yml index 6bdfa9a9d..50e33b9bb 100644 --- a/.github/workflows/centos-ci.yml +++ b/.github/workflows/centos-ci.yml @@ -88,9 +88,6 @@ jobs: - name: System tests BioDynaMo shell: bash working-directory: build - # on: - # schedule: - # - cron: "3 * * *" run: | . bin/thisbdm.sh sudo git config --system user.name "Test User" diff --git a/.github/workflows/macos-ci.yml b/.github/workflows/macos-ci.yml index 3e9fff3d1..e574bf6fc 100644 --- a/.github/workflows/macos-ci.yml +++ b/.github/workflows/macos-ci.yml @@ -66,9 +66,6 @@ jobs: - name: System tests BioDynaMo shell: bash working-directory: build - on: - schedule: - - cron: "3 * * *" run: | . bin/thisbdm.sh git config --system user.name "Test User" diff --git a/.github/workflows/ubuntu-ci.yml b/.github/workflows/ubuntu-ci.yml index 5109663b2..2b8202b82 100644 --- a/.github/workflows/ubuntu-ci.yml +++ b/.github/workflows/ubuntu-ci.yml @@ -76,9 +76,6 @@ jobs: - name: System tests BioDynaMo shell: bash working-directory: build - on: - schedule: - - cron: "3 * * *" run: | . bin/thisbdm.sh sudo git config --system user.name "Test User" From a1c4bef0df9b941d0d519082f2adb8e1bdb66e47 Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Fri, 12 Jun 2020 02:00:45 +0200 Subject: [PATCH 13/39] Remove redundant line --- .github/workflows/ubuntu-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ubuntu-ci.yml b/.github/workflows/ubuntu-ci.yml index 2b8202b82..b7277836e 100644 --- a/.github/workflows/ubuntu-ci.yml +++ b/.github/workflows/ubuntu-ci.yml @@ -93,7 +93,6 @@ jobs: make website - name: Deploy biodynamo.org - shell: bash uses: peaceiris/actions-gh-pages@v3 with: deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }} From 3d54bef08a65356a3fe58120b54831aa9906e40e Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Fri, 12 Jun 2020 02:15:57 +0200 Subject: [PATCH 14/39] Add cmake option to enable website --- .github/workflows/ubuntu-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ubuntu-ci.yml b/.github/workflows/ubuntu-ci.yml index b7277836e..f4d6cdc01 100644 --- a/.github/workflows/ubuntu-ci.yml +++ b/.github/workflows/ubuntu-ci.yml @@ -88,7 +88,9 @@ jobs: - name: Build biodynamo.org shell: bash working-directory: build + if: matrix.os == 'ubuntu-20.04' run: | + cmake -Dwebsite=on .. . bin/thisbdm.sh make website From 0cb0528e66c238a3c0fb6a2ce4196d125e26f92e Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Fri, 12 Jun 2020 02:30:55 +0200 Subject: [PATCH 15/39] Make separate website workflow --- .github/workflows/ubuntu-ci.yml | 20 ------------- .github/workflows/website-ci.yml | 48 ++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/website-ci.yml diff --git a/.github/workflows/ubuntu-ci.yml b/.github/workflows/ubuntu-ci.yml index f4d6cdc01..c30aa9899 100644 --- a/.github/workflows/ubuntu-ci.yml +++ b/.github/workflows/ubuntu-ci.yml @@ -85,26 +85,6 @@ jobs: ninja run-demos ../util/xvfb-initd.sh stop - - name: Build biodynamo.org - shell: bash - working-directory: build - if: matrix.os == 'ubuntu-20.04' - run: | - cmake -Dwebsite=on .. - . bin/thisbdm.sh - make website - - - name: Deploy biodynamo.org - uses: peaceiris/actions-gh-pages@v3 - with: - deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }} - external_repository: BioDynaMo/biodynamo.github.io - publish_branch: master - user_name: 'github-actions[bot]' - user_email: 'bdmtravis@gmail.com' - commit_message: ${{ github.event.head_commit.message }} - publish_dir: ./website/public - - name: Notify Slack uses: 8398a7/action-slack@v3 with: diff --git a/.github/workflows/website-ci.yml b/.github/workflows/website-ci.yml new file mode 100644 index 000000000..c51003f52 --- /dev/null +++ b/.github/workflows/website-ci.yml @@ -0,0 +1,48 @@ +name: Website CI + +on: + push: + pull_request: + branches: + - master + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout BioDynaMo + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Build website + shell: bash + run: | + cmake -G Ninja \ + -Dwebsite=on + -B build + cd build + ninja website + + - name: Deploy biodynamo.org + uses: peaceiris/actions-gh-pages@v3 + with: + deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }} + external_repository: BioDynaMo/biodynamo.github.io + publish_branch: master + user_name: 'github-actions[bot]' + user_email: 'bdmtravis@gmail.com' + commit_message: ${{ github.event.head_commit.message }} + publish_dir: ./build/website/public + + - name: Notify Slack + uses: 8398a7/action-slack@v3 + with: + status: ${{ job.status }} + author_name: Integration Test + fields: repo,ref,message,author,action,workflow # default: repo,commit + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # optional + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} # required + if: always() # Pick up events even if the job fails or is canceled. From 0b90e1c29b351a8e777d9f4c7f52548cab8a5bf3 Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Fri, 12 Jun 2020 02:32:59 +0200 Subject: [PATCH 16/39] Add dependencies installation --- .github/workflows/website-ci.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/website-ci.yml b/.github/workflows/website-ci.yml index c51003f52..fa3ca9d3d 100644 --- a/.github/workflows/website-ci.yml +++ b/.github/workflows/website-ci.yml @@ -11,6 +11,14 @@ jobs: runs-on: ubuntu-latest steps: + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y ninja-build + curl -L -O https://github.com/Kitware/CMake/releases/download/v3.17.3/cmake-3.17.3-Linux-x86_64.sh + chmod +x cmake-3.17.3-Linux-x86_64.sh + sudo ./cmake-3.17.3-Linux-x86_64.sh --skip-license --prefix=/usr/local + - name: Checkout BioDynaMo uses: actions/checkout@v2 with: From 227fda70af8afc9d34e38716b07872183f69124a Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Fri, 12 Jun 2020 02:35:00 +0200 Subject: [PATCH 17/39] Add pyenv steps --- .github/workflows/website-ci.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/website-ci.yml b/.github/workflows/website-ci.yml index fa3ca9d3d..8f2b66339 100644 --- a/.github/workflows/website-ci.yml +++ b/.github/workflows/website-ci.yml @@ -19,6 +19,22 @@ jobs: chmod +x cmake-3.17.3-Linux-x86_64.sh sudo ./cmake-3.17.3-Linux-x86_64.sh --skip-license --prefix=/usr/local + - name: Cache pyenv + uses: actions/cache@v2 + id: pyenv-cache + with: + path: ~/.pyenv + key: ${{ env.OSVERS }}-pyenv-3.6.9 + + - name: Install pyenv dependency + if: steps.pyenv-cache.outputs.cache-hit != 'true' + run: | + curl https://pyenv.run | bash + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install 3.6.9 + pyenv shell 3.6.9 + - name: Checkout BioDynaMo uses: actions/checkout@v2 with: From a720a05536278b669f260f5ccc39eaa0a7bc71c5 Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Fri, 12 Jun 2020 02:40:09 +0200 Subject: [PATCH 18/39] Install doxygen --- .github/workflows/website-ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/website-ci.yml b/.github/workflows/website-ci.yml index 8f2b66339..aec5e5145 100644 --- a/.github/workflows/website-ci.yml +++ b/.github/workflows/website-ci.yml @@ -14,11 +14,14 @@ jobs: - name: Install dependencies run: | sudo apt-get update - sudo apt-get install -y ninja-build + sudo apt-get install -y ninja-build doxygen curl -L -O https://github.com/Kitware/CMake/releases/download/v3.17.3/cmake-3.17.3-Linux-x86_64.sh chmod +x cmake-3.17.3-Linux-x86_64.sh sudo ./cmake-3.17.3-Linux-x86_64.sh --skip-license --prefix=/usr/local + - name: Set OSVERS + run: echo "::set-env name=OSVERS::$(lsb_release -si)-$(lsb_release -sr)" + - name: Cache pyenv uses: actions/cache@v2 id: pyenv-cache From cfbfc78227ed03302ade6357e9d421075d8a9fd0 Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Fri, 12 Jun 2020 02:43:34 +0200 Subject: [PATCH 19/39] Syntax error --- .github/workflows/website-ci.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/website-ci.yml b/.github/workflows/website-ci.yml index aec5e5145..fb00e0f2c 100644 --- a/.github/workflows/website-ci.yml +++ b/.github/workflows/website-ci.yml @@ -43,11 +43,17 @@ jobs: with: fetch-depth: 0 + - name: Cache Third Party Packages + uses: actions/cache@v2 + with: + path: build/third_party + key: ${{ env.OSVERS }}-third-party-${{ hashFiles('cmake/external/SHA256Digests.cmake') }} + - name: Build website shell: bash run: | cmake -G Ninja \ - -Dwebsite=on + -Dwebsite=on \ -B build cd build ninja website From 08a26c4e49fca3d453b09b16797f75e3e19fe581 Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Fri, 12 Jun 2020 02:55:40 +0200 Subject: [PATCH 20/39] Fix website cmake file --- .github/workflows/website-ci.yml | 2 +- cmake/Website.cmake | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/website-ci.yml b/.github/workflows/website-ci.yml index fb00e0f2c..eb40f7379 100644 --- a/.github/workflows/website-ci.yml +++ b/.github/workflows/website-ci.yml @@ -14,7 +14,7 @@ jobs: - name: Install dependencies run: | sudo apt-get update - sudo apt-get install -y ninja-build doxygen + sudo apt-get install -y ninja-build doxygen graphviz curl -L -O https://github.com/Kitware/CMake/releases/download/v3.17.3/cmake-3.17.3-Linux-x86_64.sh chmod +x cmake-3.17.3-Linux-x86_64.sh sudo ./cmake-3.17.3-Linux-x86_64.sh --skip-license --prefix=/usr/local diff --git a/cmake/Website.cmake b/cmake/Website.cmake index 54d68e9fa..4b98bf0f4 100644 --- a/cmake/Website.cmake +++ b/cmake/Website.cmake @@ -19,7 +19,6 @@ function(GenerateStaticFiles TARGET FLAGS) file(MAKE_DIRECTORY "${WEB_DIR}") add_custom_target(${TARGET} WORKING_DIRECTORY "${WEB_DIR}" - COMMAND rm -rf ${WEB_DIR} && mkdir ${WEB_DIR} COMMAND git clone https://github.com/BioDynaMo/website.git . # TODO: merge into master to avoid this COMMAND git checkout web-updates From 870ee329762305606fe3894ce663ad3b617109e1 Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Fri, 12 Jun 2020 11:26:24 +0200 Subject: [PATCH 21/39] Fix failing centos build --- .github/workflows/centos-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/centos-ci.yml b/.github/workflows/centos-ci.yml index 50e33b9bb..dae992175 100644 --- a/.github/workflows/centos-ci.yml +++ b/.github/workflows/centos-ci.yml @@ -89,6 +89,9 @@ jobs: shell: bash working-directory: build run: | + . scl_source enable devtoolset-7 || true + . /etc/profile.d/modules.sh || true + module load mpi . bin/thisbdm.sh sudo git config --system user.name "Test User" sudo git config --system user.email user@test.com From 952288aa8edd3c0ba13c453db31df27df21e2aa1 Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Fri, 12 Jun 2020 11:37:08 +0200 Subject: [PATCH 22/39] Remove sudo for git commands --- .github/workflows/centos-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/centos-ci.yml b/.github/workflows/centos-ci.yml index dae992175..1dac86e67 100644 --- a/.github/workflows/centos-ci.yml +++ b/.github/workflows/centos-ci.yml @@ -93,8 +93,8 @@ jobs: . /etc/profile.d/modules.sh || true module load mpi . bin/thisbdm.sh - sudo git config --system user.name "Test User" - sudo git config --system user.email user@test.com + git config --system user.name "Test User" + git config --system user.email user@test.com export DISPLAY=:99.0 ../util/xvfb-initd.sh start ninja run-demos From f16b1685875eaf81029b8eb7a7e371ef70f81ae6 Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Tue, 16 Jun 2020 13:36:11 +0200 Subject: [PATCH 23/39] Test valgrind targets --- .github/workflows/centos-ci.yml | 10 ++++++++++ .github/workflows/ubuntu-ci.yml | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/.github/workflows/centos-ci.yml b/.github/workflows/centos-ci.yml index 1dac86e67..f3e1f8b44 100644 --- a/.github/workflows/centos-ci.yml +++ b/.github/workflows/centos-ci.yml @@ -85,6 +85,16 @@ jobs: ninja run-unit-tests ../util/xvfb-initd.sh stop + - name: Valgrind tests BioDynaMo + shell: bash + working-directory: build + run: | + . bin/thisbdm.sh + export DISPLAY=:99.0 + ../util/xvfb-initd.sh start + ninja run-valgrind + ../util/xvfb-initd.sh stop + - name: System tests BioDynaMo shell: bash working-directory: build diff --git a/.github/workflows/ubuntu-ci.yml b/.github/workflows/ubuntu-ci.yml index c30aa9899..92319ee36 100644 --- a/.github/workflows/ubuntu-ci.yml +++ b/.github/workflows/ubuntu-ci.yml @@ -73,6 +73,16 @@ jobs: ninja run-unit-tests ../util/xvfb-initd.sh stop + - name: Valgrind tests BioDynaMo + shell: bash + working-directory: build + run: | + . bin/thisbdm.sh + export DISPLAY=:99.0 + ../util/xvfb-initd.sh start + ninja run-valgrind + ../util/xvfb-initd.sh stop + - name: System tests BioDynaMo shell: bash working-directory: build From 3d7c65cef566b413446c4640f856c4ee1b4dcc8e Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Tue, 16 Jun 2020 13:49:20 +0200 Subject: [PATCH 24/39] Add CentOS specific shell commands first.. --- .github/workflows/centos-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/centos-ci.yml b/.github/workflows/centos-ci.yml index f3e1f8b44..cf40687aa 100644 --- a/.github/workflows/centos-ci.yml +++ b/.github/workflows/centos-ci.yml @@ -89,6 +89,9 @@ jobs: shell: bash working-directory: build run: | + . scl_source enable devtoolset-7 || true + . /etc/profile.d/modules.sh || true + module load mpi . bin/thisbdm.sh export DISPLAY=:99.0 ../util/xvfb-initd.sh start From 40509a166e832a487dbd2d551e4f3835e5f86da2 Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Wed, 17 Jun 2020 09:51:04 +0200 Subject: [PATCH 25/39] Exclude diffusion convergence test from valgrind These take very long, and would not expose any new memory leaks, in addition to the already existing diffusion tests --- cmake/Testing.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/Testing.cmake b/cmake/Testing.cmake index b2ec44ab9..1d35de9c6 100644 --- a/cmake/Testing.cmake +++ b/cmake/Testing.cmake @@ -81,9 +81,9 @@ function(bdm_add_test_executable TEST_TARGET) if (valgrind AND VALGRIND_FOUND AND NOT coverage) # filter out SchedulerTest.Backup because of timing issue add_test(NAME "valgrind_${TEST_TARGET}" - COMMAND ${CMAKE_BINARY_DIR}/launcher.sh ${CMAKE_SOURCE_DIR}/util/valgrind.sh ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TEST_TARGET} -- --gtest_filter=-*DeathTest.*:IOTest.InvalidRead:SchedulerTest.Backup:ResourceManagerTest.SortAndApplyOnAllElementsParallel*:InlineVector*:NeuriteElementBehaviour.*:MechanicalInteraction.*) + COMMAND ${CMAKE_BINARY_DIR}/launcher.sh ${CMAKE_SOURCE_DIR}/util/valgrind.sh ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TEST_TARGET} -- --gtest_filter=-*DeathTest.*:IOTest.InvalidRead:SchedulerTest.Backup:ResourceManagerTest.SortAndApplyOnAllElementsParallel*:InlineVector*:NeuriteElementBehaviour.*:MechanicalInteraction.*:DiffusionTest.*Convergence*) add_custom_target(run-valgrind - COMMAND ${CMAKE_BINARY_DIR}/launcher.sh ${CMAKE_SOURCE_DIR}/util/valgrind.sh ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TEST_TARGET} -- --gtest_filter=-*DeathTest.*:IOTest.InvalidRead:SchedulerTest.Backup:ResourceManagerTest.SortAndApplyOnAllElementsParallel*:InlineVector*:NeuriteElementBehaviour.*:MechanicalInteraction.*) + COMMAND ${CMAKE_BINARY_DIR}/launcher.sh ${CMAKE_SOURCE_DIR}/util/valgrind.sh ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${TEST_TARGET} -- --gtest_filter=-*DeathTest.*:IOTest.InvalidRead:SchedulerTest.Backup:ResourceManagerTest.SortAndApplyOnAllElementsParallel*:InlineVector*:NeuriteElementBehaviour.*:MechanicalInteraction.*:DiffusionTest.*Convergence*) add_dependencies(run-valgrind biodynamo-unit-tests) endif() From fd58c5ff0962137a131d959f0113f91cf3a85afd Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Wed, 17 Jun 2020 09:51:25 +0200 Subject: [PATCH 26/39] Initialize boolean --- test/unit/core/sim_object/cell_test.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/core/sim_object/cell_test.h b/test/unit/core/sim_object/cell_test.h index b571db6a5..fb992413c 100644 --- a/test/unit/core/sim_object/cell_test.h +++ b/test/unit/core/sim_object/cell_test.h @@ -65,7 +65,7 @@ class TestCell : public Cell { double captured_phi_ = 0.0; double captured_theta_ = 0.0; - bool placeholder_; // BDM_SIM_OBJECT_HEADER needs at least one member + bool placeholder_ = false; // BDM_SIM_OBJECT_HEADER needs at least one member FRIEND_TEST(CellTest, DivideVolumeRatioPhiTheta); }; From c313ed23e53db5f4ebb4785d64db74fc30f5fc0c Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Wed, 17 Jun 2020 10:12:57 +0200 Subject: [PATCH 27/39] Update valgrind suppression file * Generalize suppression of errors coming from vtk libraries. * Add new suppression rules for newer valgrind releases --- util/valgrind-bdm.supp | 376 +++-------------------------------------- 1 file changed, 19 insertions(+), 357 deletions(-) diff --git a/util/valgrind-bdm.supp b/util/valgrind-bdm.supp index 5dc35ce80..b61dfd4ca 100644 --- a/util/valgrind-bdm.supp +++ b/util/valgrind-bdm.supp @@ -293,6 +293,24 @@ fun:getpwuid_r@@GLIBC_2.2.5 } +######### Suppress OpenMPI issues +{ + ignore_openmpi_false_positives + Memcheck:Leak + ... + obj:*/openmpi/*.so* + ... +} + +######### Suppress libevent issues +{ + ignore_libevent_false_positives + Memcheck:Leak + ... + obj:*/libevent*.so* + ... +} + ######### Suppress OpenCL issues { ignore_opencl_false_positives @@ -347,368 +365,12 @@ ######### Suppress Paraview -{ - paraview-1 - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:opal_db_base_store - fun:ompi_modex_send_string - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN16vtkProcessModule10InitializeENS_12ProcessTypesERiRPPc - fun:_ZN23vtkInitializationHelper10InitializeEiPPciP12vtkPVOptions - fun:_ZN14vtkCPCxxHelper3NewEv - fun:_ZN14vtkCPProcessor10InitializeEPKc - fun:_ZN3bdm15CatalystAdaptor10InitializeEv - fun:_ZN3bdm15CatalystAdaptor9VisualizeEmb -} - -{ - paraview-2 - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:strdup - obj:* - fun:opal_db_base_store - obj:* - fun:orte_init - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN16vtkProcessModule10InitializeENS_12ProcessTypesERiRPPc - fun:_ZN23vtkInitializationHelper10InitializeEiPPciP12vtkPVOptions - fun:_ZN14vtkCPCxxHelper3NewEv - fun:_ZN14vtkCPProcessor10InitializeEPKc -} - -{ - paraview-3 - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - fun:strdup - obj:* - obj:* - obj:* - fun:hwloc_topology_load - fun:opal_hwloc_base_get_topology - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN16vtkProcessModule10InitializeENS_12ProcessTypesERiRPPc - fun:_ZN23vtkInitializationHelper10InitializeEiPPciP12vtkPVOptions - fun:_ZN14vtkCPCxxHelper3NewEv -} - -{ - paraview-4 - Memcheck:Leak - match-leak-kinds: indirect - fun:calloc - obj:* - obj:* - obj:* - fun:hwloc_topology_load - fun:opal_hwloc_base_get_topology - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN16vtkProcessModule10InitializeENS_12ProcessTypesERiRPPc - fun:_ZN23vtkInitializationHelper10InitializeEiPPciP12vtkPVOptions - fun:_ZN14vtkCPCxxHelper3NewEv - fun:_ZN14vtkCPProcessor10InitializeEPKc -} - -{ - paraview-5 - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:lt__malloc - fun:lt__zalloc - obj:/usr/lib/x86_64-linux-gnu/libltdl.so.7.3.1 - fun:lt_dlopenadvise - obj:* - obj:/usr/lib/x86_64-linux-gnu/libhwloc.so.5.6.8 - fun:hwloc_topology_init - fun:opal_hwloc_base_get_topology - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN16vtkProcessModule10InitializeENS_12ProcessTypesERiRPPc -} - -{ - paraview-6 - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - fun:__vasprintf_chk - fun:__asprintf_chk - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN16vtkProcessModule10InitializeENS_12ProcessTypesERiRPPc - fun:_ZN23vtkInitializationHelper10InitializeEiPPciP12vtkPVOptions - fun:_ZN14vtkCPCxxHelper3NewEv - fun:_ZN14vtkCPProcessor10InitializeEPKc - fun:_ZN3bdm15CatalystAdaptor10InitializeEv - fun:_ZN3bdm15CatalystAdaptor9VisualizeEmb -} - -{ - paraview-7 - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - fun:opal_db_base_store - fun:ompi_modex_send - obj:* - fun:mca_btl_base_select - obj:* - fun:mca_bml_base_init - obj:* - fun:mca_pml_base_select - fun:ompi_mpi_init - fun:PMPI_Init -} - -{ - paraview-8 - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - fun:realloc - fun:mca_bml_base_btl_array_reserve - obj:* - obj:* - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN16vtkProcessModule10InitializeENS_12ProcessTypesERiRPPc - fun:_ZN23vtkInitializationHelper10InitializeEiPPciP12vtkPVOptions - fun:_ZN14vtkCPCxxHelper3NewEv - fun:_ZN14vtkCPProcessor10InitializeEPKc - fun:_ZN3bdm15CatalystAdaptor10InitializeEv -} - -{ - paraview-9 - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - fun:realloc - fun:mca_bml_base_btl_array_reserve - obj:* - obj:* - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN16vtkProcessModule10InitializeENS_12ProcessTypesERiRPPc - fun:_ZN23vtkInitializationHelper10InitializeEiPPciP12vtkPVOptions - fun:_ZN14vtkCPCxxHelper3NewEv - fun:_ZN14vtkCPProcessor10InitializeEPKc - fun:_ZN3bdm15CatalystAdaptor10InitializeEv -} - -{ - paraview-10 - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - fun:realloc - fun:mca_bml_base_btl_array_reserve - obj:* - obj:* - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN16vtkProcessModule10InitializeENS_12ProcessTypesERiRPPc - fun:_ZN23vtkInitializationHelper10InitializeEiPPciP12vtkPVOptions - fun:_ZN14vtkCPCxxHelper3NewEv - fun:_ZN14vtkCPProcessor10InitializeEPKc - fun:_ZN3bdm15CatalystAdaptor10InitializeEv -} - -{ - paraview-11 - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - fun:strdup - obj:* - obj:* - obj:* - obj:* - fun:hwloc_topology_load - fun:opal_hwloc_base_get_topology - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN16vtkProcessModule10InitializeENS_12ProcessTypesERiRPPc - fun:_ZN23vtkInitializationHelper10InitializeEiPPciP12vtkPVOptions -} - -{ - paraview-12 - Memcheck:Leak - match-leak-kinds: definite - fun:malloc - obj:* - obj:* - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN16vtkProcessModule10InitializeENS_12ProcessTypesERiRPPc - fun:_ZN23vtkInitializationHelper10InitializeEiPPciP12vtkPVOptions - fun:_ZN14vtkCPCxxHelper3NewEv - fun:_ZN14vtkCPProcessor10InitializeEPKc - fun:_ZN3bdm15CatalystAdaptor10InitializeEv - fun:_ZN3bdm15CatalystAdaptor9VisualizeEmb -} - -{ - paraview-13 - Memcheck:Leak - match-leak-kinds: indirect - fun:calloc - obj:* - obj:* - obj:* - fun:hwloc_topology_load - fun:opal_hwloc_base_get_topology - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN16vtkProcessModule10InitializeENS_12ProcessTypesERiRPPc - fun:_ZN23vtkInitializationHelper10InitializeEiPPciP12vtkPVOptions - fun:_ZN14vtkCPCxxHelper3NewEv - fun:_ZN14vtkCPProcessor10InitializeEPKc -} - -{ - paraview-14 - Memcheck:Leak - match-leak-kinds: indirect - fun:malloc - fun:strdup - obj:* - obj:* - obj:* - obj:* - fun:hwloc_topology_load - fun:opal_hwloc_base_get_topology - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN16vtkProcessModule10InitializeENS_12ProcessTypesERiRPPc - fun:_ZN23vtkInitializationHelper10InitializeEiPPciP12vtkPVOptions -} - -{ - paraview-15 - Memcheck:Leak - match-leak-kinds: indirect - fun:realloc - obj:* - obj:* - obj:* - obj:* - fun:hwloc_topology_load - fun:opal_hwloc_base_get_topology - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN16vtkProcessModule10InitializeENS_12ProcessTypesERiRPPc - fun:_ZN23vtkInitializationHelper10InitializeEiPPciP12vtkPVOptions - fun:_ZN14vtkCPCxxHelper3NewEv -} - -{ - paraview-16 - Memcheck:Leak - match-leak-kinds: definite - fun:calloc - obj:* - obj:* - obj:* - fun:hwloc_topology_load - fun:opal_hwloc_base_get_topology - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN16vtkProcessModule10InitializeENS_12ProcessTypesERiRPPc - fun:_ZN23vtkInitializationHelper10InitializeEiPPciP12vtkPVOptions - fun:_ZN14vtkCPCxxHelper3NewEv - fun:_ZN14vtkCPProcessor10InitializeEPKc -} - -{ - paraview-17 - Memcheck:Leak - match-leak-kinds: possible - fun:malloc - fun:strdup - obj:* - obj:* - obj:* - obj:* - fun:hwloc_topology_load - fun:opal_hwloc_base_get_topology - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN16vtkProcessModule10InitializeENS_12ProcessTypesERiRPPc - fun:_ZN23vtkInitializationHelper10InitializeEiPPciP12vtkPVOptions -} - -{ - paraview-18 - Memcheck:Leak - match-leak-kinds: possible - fun:malloc - fun:strdup - obj:* - obj:* - obj:* - obj:* - fun:hwloc_topology_load - fun:opal_hwloc_base_get_topology - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN16vtkProcessModule10InitializeENS_12ProcessTypesERiRPPc - fun:_ZN23vtkInitializationHelper10InitializeEiPPciP12vtkPVOptions -} - -{ - paraview-19 - Memcheck:Leak - match-leak-kinds: possible - fun:realloc - obj:* - obj:* - obj:* - obj:* - fun:hwloc_topology_load - fun:opal_hwloc_base_get_topology - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN16vtkProcessModule10InitializeENS_12ProcessTypesERiRPPc - fun:_ZN23vtkInitializationHelper10InitializeEiPPciP12vtkPVOptions - fun:_ZN14vtkCPCxxHelper3NewEv -} - -{ - paraview-20 - Memcheck:Cond - fun:vfprintf - fun:__vasprintf_chk - fun:__asprintf_chk - fun:pml_v_output_open - obj:/usr/lib/openmpi/lib/libmpi.so.12.0.2 - fun:mca_base_framework_components_open - obj:/usr/lib/openmpi/lib/libmpi.so.12.0.2 - fun:mca_base_framework_open - fun:ompi_mpi_init - fun:PMPI_Init - fun:_ZN16vtkProcessModule10InitializeENS_12ProcessTypesERiRPPc - fun:_ZN23vtkInitializationHelper10InitializeEiPPciP12vtkPVOptions -} { paraview-catalyst Memcheck:Leak ... - obj:*/libvtkPVCatalyst-pv*.so.* + obj:*/libvtk*.so.* } ######### Suppress TBB issues From 4dc488d7f3566dae8642f3b8ee11f0844317179c Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Wed, 17 Jun 2020 10:25:20 +0200 Subject: [PATCH 28/39] Install valgrind as prerequisite --- .github/workflows/centos-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/centos-ci.yml b/.github/workflows/centos-ci.yml index cf40687aa..5a262220e 100644 --- a/.github/workflows/centos-ci.yml +++ b/.github/workflows/centos-ci.yml @@ -19,7 +19,7 @@ jobs: yum -y install centos-release-scl epel-release yum -y install man wget environment-modules libXt-devel libXext-devel ninja-build make \ devtoolset-7-gcc* numactl-devel openmpi3-devel freeglut-devel redhat-lsb-core \ - scl-utils which xorg-x11-server-Xvfb fontconfig mesa-dri-drivers + scl-utils which xorg-x11-server-Xvfb fontconfig mesa-dri-drivers valgrind yum -y install @development zlib-devel bzip2 bzip2-devel readline-devel sqlite \ sqlite-devel openssl-devel xz xz-devel libffi-devel findutils yum -y remove git* From 1366d4516c63486454d620b17296c6264f4eaebb Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Wed, 17 Jun 2020 12:19:05 +0200 Subject: [PATCH 29/39] Allow longer list of call stack for valgrind --- util/valgrind.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/util/valgrind.sh b/util/valgrind.sh index c400aac98..121edb1f0 100755 --- a/util/valgrind.sh +++ b/util/valgrind.sh @@ -17,6 +17,7 @@ BDM_PROJECT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.." valgrind \ --track-origins=yes \ + --num-callers=50 \ --leak-resolution=high \ --tool=memcheck \ --leak-check=full \ From 8114f432ad113df3f21efceaa7e83f693537eb49 Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Thu, 18 Jun 2020 01:05:36 +0200 Subject: [PATCH 30/39] Add ROOT suppression file to valgrind script --- util/valgrind.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/util/valgrind.sh b/util/valgrind.sh index 121edb1f0..e5157e7fc 100755 --- a/util/valgrind.sh +++ b/util/valgrind.sh @@ -25,5 +25,6 @@ valgrind \ --gen-suppressions=all \ --show-reachable=no \ --suppressions=${BDM_PROJECT_DIR}/util/valgrind-bdm.supp \ + --suppressions=$ROOTSYS/etc/valgrind-root.supp \ --error-exitcode=1 \ $@ From c7fc95c6337ac199df8ae487bbbc1b2fa2bad7c8 Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Thu, 18 Jun 2020 11:53:45 +0200 Subject: [PATCH 31/39] Suppress valgrind false positives on GH Actions --- util/valgrind-bdm.supp | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/util/valgrind-bdm.supp b/util/valgrind-bdm.supp index b61dfd4ca..18326b9fa 100644 --- a/util/valgrind-bdm.supp +++ b/util/valgrind-bdm.supp @@ -244,6 +244,14 @@ ... } +{ + false_positive_root_3 + Memcheck:Param + ... + fun:cling_runtime_internal_throwIfInvalidPointer + ... +} + ######### Suppress protobuf library issues { ignore_protobuf_memleaks @@ -302,6 +310,14 @@ ... } +{ + ignore_openmpi_false_positives_cond + Memcheck:Cond + ... + obj:*/openmpi/*.so* + ... +} + ######### Suppress libevent issues { ignore_libevent_false_positives From 85156ed899c6a68fd86fd79d33fdbc72483376f6 Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Thu, 18 Jun 2020 12:08:25 +0200 Subject: [PATCH 32/39] Fix suppression syntax error --- util/valgrind-bdm.supp | 1 + 1 file changed, 1 insertion(+) diff --git a/util/valgrind-bdm.supp b/util/valgrind-bdm.supp index 18326b9fa..55f513ad2 100644 --- a/util/valgrind-bdm.supp +++ b/util/valgrind-bdm.supp @@ -247,6 +247,7 @@ { false_positive_root_3 Memcheck:Param + msync(start) ... fun:cling_runtime_internal_throwIfInvalidPointer ... From 03667d70ef4df03acccad188e86758b2dfb970a7 Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Thu, 18 Jun 2020 12:42:16 +0200 Subject: [PATCH 33/39] Avoid usage of static members in gtests --- test/unit/test_util/io_test.h | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/test/unit/test_util/io_test.h b/test/unit/test_util/io_test.h index f707a215c..fdbebb849 100644 --- a/test/unit/test_util/io_test.h +++ b/test/unit/test_util/io_test.h @@ -23,6 +23,9 @@ #include "TBufferJSON.h" +#define ROOT_FILE "io-test.root" +#define JSON_FILE "io-test.json" + namespace bdm { /// Test fixture for io tests that follow the same form @@ -41,11 +44,7 @@ namespace bdm { /// EXPECT_EQ(..., restored->GetDataMember1()); /// ... /// } -class IOTest : public ::testing::Test { - public: - static constexpr char const* kRootFile = "io-test.root"; - static constexpr char const* kJsonFile = "io-test.json"; -}; +class IOTest : public ::testing::Test {}; /// Writes backup to file and reads it back into restored /// Outside the test fixture so it can be called in a function from the header. @@ -53,19 +52,19 @@ class IOTest : public ::testing::Test { /// error and must be placed in a source file. template void BackupAndRestore(const T& backup, T** restored) { - remove(IOTest::kRootFile); - remove(IOTest::kJsonFile); + remove(ROOT_FILE); + remove(JSON_FILE); // write to root file - WritePersistentObject(IOTest::kRootFile, "T", backup, "new"); + WritePersistentObject(ROOT_FILE, "T", backup, "new"); // Two benefits of writing an object to JSON: // 1) Ensures that a dictionary must exists; otherwise linking error // 2) Print out the object's content for debugging purposes - TBufferJSON::ExportToFile(IOTest::kJsonFile, &backup, backup.Class()); + TBufferJSON::ExportToFile(JSON_FILE, &backup, backup.Class()); // read back - GetPersistentObject(IOTest::kRootFile, "T", *restored); + GetPersistentObject(ROOT_FILE, "T", *restored); } } // namespace bdm From 3f90e87f5925a939500618377c08c09090667e59 Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Thu, 18 Jun 2020 13:23:24 +0200 Subject: [PATCH 34/39] Add valgrind false positive suppression on centOS --- util/valgrind-bdm.supp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/util/valgrind-bdm.supp b/util/valgrind-bdm.supp index 55f513ad2..677a6c146 100644 --- a/util/valgrind-bdm.supp +++ b/util/valgrind-bdm.supp @@ -253,6 +253,14 @@ ... } +{ + false_positive_root_4 + Memcheck:Overlap + ... + fun:CreateInterpreter + ... +} + ######### Suppress protobuf library issues { ignore_protobuf_memleaks From 6d7bd9026bfe421cdc0d7c3c3c0d8e40655c9121 Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Thu, 18 Jun 2020 15:22:51 +0200 Subject: [PATCH 35/39] Add suppression rule for OpenMPI fake positives --- util/valgrind-bdm.supp | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/util/valgrind-bdm.supp b/util/valgrind-bdm.supp index 677a6c146..9fa4c3816 100644 --- a/util/valgrind-bdm.supp +++ b/util/valgrind-bdm.supp @@ -327,6 +327,18 @@ ... } +{ + gh_actions_centos_false_positives + Memcheck:Leak + match-leak-kinds: indirect + fun:malloc + ... + fun:opal_libevent2022_event_base_loop + ... + fun:start_thread + fun:clone +} + ######### Suppress libevent issues { ignore_libevent_false_positives From bfbf956a29bd14bbd4ce57d50f1ed775c486f32b Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Thu, 18 Jun 2020 15:23:55 +0200 Subject: [PATCH 36/39] Remove website workflow. To be added to gh pages repo --- .github/workflows/website-ci.yml | 81 -------------------------------- 1 file changed, 81 deletions(-) delete mode 100644 .github/workflows/website-ci.yml diff --git a/.github/workflows/website-ci.yml b/.github/workflows/website-ci.yml deleted file mode 100644 index eb40f7379..000000000 --- a/.github/workflows/website-ci.yml +++ /dev/null @@ -1,81 +0,0 @@ -name: Website CI - -on: - push: - pull_request: - branches: - - master - -jobs: - build: - runs-on: ubuntu-latest - - steps: - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y ninja-build doxygen graphviz - curl -L -O https://github.com/Kitware/CMake/releases/download/v3.17.3/cmake-3.17.3-Linux-x86_64.sh - chmod +x cmake-3.17.3-Linux-x86_64.sh - sudo ./cmake-3.17.3-Linux-x86_64.sh --skip-license --prefix=/usr/local - - - name: Set OSVERS - run: echo "::set-env name=OSVERS::$(lsb_release -si)-$(lsb_release -sr)" - - - name: Cache pyenv - uses: actions/cache@v2 - id: pyenv-cache - with: - path: ~/.pyenv - key: ${{ env.OSVERS }}-pyenv-3.6.9 - - - name: Install pyenv dependency - if: steps.pyenv-cache.outputs.cache-hit != 'true' - run: | - curl https://pyenv.run | bash - export PATH="$HOME/.pyenv/bin:$PATH" - eval "$(pyenv init -)" - PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install 3.6.9 - pyenv shell 3.6.9 - - - name: Checkout BioDynaMo - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Cache Third Party Packages - uses: actions/cache@v2 - with: - path: build/third_party - key: ${{ env.OSVERS }}-third-party-${{ hashFiles('cmake/external/SHA256Digests.cmake') }} - - - name: Build website - shell: bash - run: | - cmake -G Ninja \ - -Dwebsite=on \ - -B build - cd build - ninja website - - - name: Deploy biodynamo.org - uses: peaceiris/actions-gh-pages@v3 - with: - deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }} - external_repository: BioDynaMo/biodynamo.github.io - publish_branch: master - user_name: 'github-actions[bot]' - user_email: 'bdmtravis@gmail.com' - commit_message: ${{ github.event.head_commit.message }} - publish_dir: ./build/website/public - - - name: Notify Slack - uses: 8398a7/action-slack@v3 - with: - status: ${{ job.status }} - author_name: Integration Test - fields: repo,ref,message,author,action,workflow # default: repo,commit - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # optional - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} # required - if: always() # Pick up events even if the job fails or is canceled. From 404abd49ef97f89ddae5af3ee60fd771a11bfe30 Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Thu, 18 Jun 2020 15:49:15 +0200 Subject: [PATCH 37/39] All leak-kinds... --- util/valgrind-bdm.supp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/valgrind-bdm.supp b/util/valgrind-bdm.supp index 9fa4c3816..c805bd072 100644 --- a/util/valgrind-bdm.supp +++ b/util/valgrind-bdm.supp @@ -330,7 +330,7 @@ { gh_actions_centos_false_positives Memcheck:Leak - match-leak-kinds: indirect + match-leak-kinds: all fun:malloc ... fun:opal_libevent2022_event_base_loop From 2f0396d0e85e026e97c086cd7a575c56342bea28 Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Thu, 18 Jun 2020 16:21:33 +0200 Subject: [PATCH 38/39] Separate cron jobs from main workflow files --- .github/workflows/centos-ci.yml | 30 +------ .github/workflows/centos-system-ci.yml | 119 +++++++++++++++++++++++++ .github/workflows/macos-ci.yml | 12 --- .github/workflows/macos-system-ci.yml | 87 ++++++++++++++++++ .github/workflows/ubuntu-ci.yml | 25 +----- .github/workflows/ubuntu-system-ci.yml | 104 +++++++++++++++++++++ cmake/Website.cmake | 1 + 7 files changed, 313 insertions(+), 65 deletions(-) create mode 100644 .github/workflows/centos-system-ci.yml create mode 100644 .github/workflows/macos-system-ci.yml create mode 100644 .github/workflows/ubuntu-system-ci.yml diff --git a/.github/workflows/centos-ci.yml b/.github/workflows/centos-ci.yml index 5a262220e..592789059 100644 --- a/.github/workflows/centos-ci.yml +++ b/.github/workflows/centos-ci.yml @@ -19,7 +19,7 @@ jobs: yum -y install centos-release-scl epel-release yum -y install man wget environment-modules libXt-devel libXext-devel ninja-build make \ devtoolset-7-gcc* numactl-devel openmpi3-devel freeglut-devel redhat-lsb-core \ - scl-utils which xorg-x11-server-Xvfb fontconfig mesa-dri-drivers valgrind + scl-utils which xorg-x11-server-Xvfb fontconfig mesa-dri-drivers yum -y install @development zlib-devel bzip2 bzip2-devel readline-devel sqlite \ sqlite-devel openssl-devel xz xz-devel libffi-devel findutils yum -y remove git* @@ -85,34 +85,6 @@ jobs: ninja run-unit-tests ../util/xvfb-initd.sh stop - - name: Valgrind tests BioDynaMo - shell: bash - working-directory: build - run: | - . scl_source enable devtoolset-7 || true - . /etc/profile.d/modules.sh || true - module load mpi - . bin/thisbdm.sh - export DISPLAY=:99.0 - ../util/xvfb-initd.sh start - ninja run-valgrind - ../util/xvfb-initd.sh stop - - - name: System tests BioDynaMo - shell: bash - working-directory: build - run: | - . scl_source enable devtoolset-7 || true - . /etc/profile.d/modules.sh || true - module load mpi - . bin/thisbdm.sh - git config --system user.name "Test User" - git config --system user.email user@test.com - export DISPLAY=:99.0 - ../util/xvfb-initd.sh start - ninja run-demos - ../util/xvfb-initd.sh stop - - name: Notify Slack uses: 8398a7/action-slack@v3 with: diff --git a/.github/workflows/centos-system-ci.yml b/.github/workflows/centos-system-ci.yml new file mode 100644 index 000000000..2a627cdbc --- /dev/null +++ b/.github/workflows/centos-system-ci.yml @@ -0,0 +1,119 @@ +# Runs the installation tests and valgrind tests in the following cases: +# 1) every day at 3 am +# 2) a pull request to the master branch + +name: CentOS System CI + +on: + schedule: + # Every day at 3 am + - cron: '0 3 * * *' + pull_request: + # Upon pull requests + branches: + - master + +jobs: + build: + runs-on: ubuntu-latest + container: + image: centos:7 + + steps: + - name: Install dependencies + run: | + yum -y update + yum -y install centos-release-scl epel-release + yum -y install man wget environment-modules libXt-devel libXext-devel ninja-build make \ + devtoolset-7-gcc* numactl-devel openmpi3-devel freeglut-devel redhat-lsb-core \ + scl-utils which xorg-x11-server-Xvfb fontconfig mesa-dri-drivers valgrind + yum -y install @development zlib-devel bzip2 bzip2-devel readline-devel sqlite \ + sqlite-devel openssl-devel xz xz-devel libffi-devel findutils + yum -y remove git* + yum -y install https://packages.endpoint.com/rhel/7/os/x86_64/endpoint-repo-1.7-1.x86_64.rpm + yum -y install git + curl -L -O https://github.com/Kitware/CMake/releases/download/v3.17.3/cmake-3.17.3-Linux-x86_64.sh + chmod +x cmake-3.17.3-Linux-x86_64.sh + ./cmake-3.17.3-Linux-x86_64.sh --skip-license --prefix=/usr/local + + - name: Set OSVERS + run: echo "::set-env name=OSVERS::$(lsb_release -si)-$(lsb_release -sr)" + + - name: Cache pyenv + uses: actions/cache@v2 + id: pyenv-cache + with: + path: ~/.pyenv + key: ${{ env.OSVERS }}-pyenv-3.6.9 + + - name: Install pyenv dependency + if: steps.pyenv-cache.outputs.cache-hit != 'true' + run: | + curl https://pyenv.run | bash + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install 3.6.9 + pyenv shell 3.6.9 + + - name: Checkout BioDynaMo + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Cache Third Party Packages + uses: actions/cache@v2 + with: + path: build/third_party + key: ${{ env.OSVERS }}-third-party-${{ hashFiles('cmake/external/SHA256Digests.cmake') }} + + - name: Build BioDynaMo + shell: bash + run: | + . scl_source enable devtoolset-7 || true + . /etc/profile.d/modules.sh || true + module load mpi + cmake -G Ninja \ + -Dparaview=ON \ + -Dnuma=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -B build + cmake --build build --parallel --config Release + + - name: Valgrind tests BioDynaMo + shell: bash + working-directory: build + run: | + . scl_source enable devtoolset-7 || true + . /etc/profile.d/modules.sh || true + module load mpi + . bin/thisbdm.sh + export DISPLAY=:99.0 + ../util/xvfb-initd.sh start + ninja run-valgrind + ../util/xvfb-initd.sh stop + + - name: System tests BioDynaMo + shell: bash + working-directory: build + run: | + . scl_source enable devtoolset-7 || true + . /etc/profile.d/modules.sh || true + module load mpi + . bin/thisbdm.sh + git config --system user.name "Test User" + git config --system user.email user@test.com + export DISPLAY=:99.0 + ../util/xvfb-initd.sh start + ninja run-demos + ../util/xvfb-initd.sh stop + + - name: Notify Slack + uses: 8398a7/action-slack@v3 + with: + status: ${{ job.status }} + author_name: Integration Test + fields: repo,ref,message,author,action,workflow # default: repo,commit + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # optional + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} # required + if: always() # Pick up events even if the job fails or is canceled. diff --git a/.github/workflows/macos-ci.yml b/.github/workflows/macos-ci.yml index e574bf6fc..8de41932d 100644 --- a/.github/workflows/macos-ci.yml +++ b/.github/workflows/macos-ci.yml @@ -63,18 +63,6 @@ jobs: . bin/thisbdm.sh ninja run-unit-tests - - name: System tests BioDynaMo - shell: bash - working-directory: build - run: | - . bin/thisbdm.sh - git config --system user.name "Test User" - git config --system user.email user@test.com - export DISPLAY=:99.0 - ../util/xvfb-initd.sh start - ninja run-demos - ../util/xvfb-initd.sh stop - - name: Notify Slack uses: 8398a7/action-slack@v3 with: diff --git a/.github/workflows/macos-system-ci.yml b/.github/workflows/macos-system-ci.yml new file mode 100644 index 000000000..b50d02111 --- /dev/null +++ b/.github/workflows/macos-system-ci.yml @@ -0,0 +1,87 @@ +# Runs the installation tests in the following cases: +# 1) every day at 3 am +# 2) a pull request to the master branch + +name: macOS System CI + +on: + schedule: + # Every day at 3 am + - cron: '0 3 * * *' + pull_request: + # Upon pull requests + branches: + - master + +jobs: + build: + runs-on: macos-latest + + steps: + - name: Install dependencies + run: | + brew install libomp open-mpi git pyenv llvm wget cmake ninja + + - name: Cache pyenv + uses: actions/cache@v2 + id: pyenv-cache + with: + path: ~/.pyenv + key: ${{ runner.os }}-pyenv-3.6.9 + + - name: Install pyenv dependency + if: steps.pyenv-cache.outputs.cache-hit != 'true' + run: | + eval "$(pyenv init -)" + PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install 3.6.9 + pyenv shell 3.6.9 + + - name: Checkout BioDynaMo + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Cache Third Party Packages + uses: actions/cache@v2 + with: + path: build/third_party + key: ${{ runner.os }}-third-party-${{ hashFiles('cmake/external/SHA256Digests.cmake') }} + + - name: Build BioDynaMo + shell: bash + run: | + cmake -G Ninja \ + -Dopencl=OFF \ + -Dparaview=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_COMPILER=/usr/local/opt/llvm/bin/clang++ \ + -DCMAKE_CXX_FLAGS="-isysroot `xcrun --show-sdk-path`" \ + -DCMAKE_C_COMPILER=/usr/local/opt/llvm/bin/clang \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=TRUE \ + -DCMAKE_EXE_LINKER_FLAGS_INIT=-L/usr/local/opt/llvm/lib \ + -DCMAKE_SHARED_LINKER_FLAGS_INIT=-L/usr/local/opt/llvm/lib \ + -B build + cmake --build build --parallel --config Release + + - name: System tests BioDynaMo + shell: bash + working-directory: build + run: | + . bin/thisbdm.sh + git config --system user.name "Test User" + git config --system user.email user@test.com + export DISPLAY=:99.0 + ../util/xvfb-initd.sh start + ninja run-demos + ../util/xvfb-initd.sh stop + + - name: Notify Slack + uses: 8398a7/action-slack@v3 + with: + status: ${{ job.status }} + author_name: Integration Test + fields: repo,ref,message,author,action,workflow # default: repo,commit + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # optional + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} # required + if: always() # Pick up events even if the job fails or is canceled. diff --git a/.github/workflows/ubuntu-ci.yml b/.github/workflows/ubuntu-ci.yml index 92319ee36..a076b2553 100644 --- a/.github/workflows/ubuntu-ci.yml +++ b/.github/workflows/ubuntu-ci.yml @@ -18,8 +18,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y libopenmpi-dev libomp5 libomp-dev libnuma-dev freeglut3-dev \ - libreadline-dev libsqlite3-dev tk-dev python-openssl ninja-build \ - valgrind + libreadline-dev libsqlite3-dev tk-dev python-openssl ninja-build curl -L -O https://github.com/Kitware/CMake/releases/download/v3.17.3/cmake-3.17.3-Linux-x86_64.sh chmod +x cmake-3.17.3-Linux-x86_64.sh sudo ./cmake-3.17.3-Linux-x86_64.sh --skip-license --prefix=/usr/local @@ -73,28 +72,6 @@ jobs: ninja run-unit-tests ../util/xvfb-initd.sh stop - - name: Valgrind tests BioDynaMo - shell: bash - working-directory: build - run: | - . bin/thisbdm.sh - export DISPLAY=:99.0 - ../util/xvfb-initd.sh start - ninja run-valgrind - ../util/xvfb-initd.sh stop - - - name: System tests BioDynaMo - shell: bash - working-directory: build - run: | - . bin/thisbdm.sh - sudo git config --system user.name "Test User" - sudo git config --system user.email user@test.com - export DISPLAY=:99.0 - ../util/xvfb-initd.sh start - ninja run-demos - ../util/xvfb-initd.sh stop - - name: Notify Slack uses: 8398a7/action-slack@v3 with: diff --git a/.github/workflows/ubuntu-system-ci.yml b/.github/workflows/ubuntu-system-ci.yml new file mode 100644 index 000000000..b418c20e5 --- /dev/null +++ b/.github/workflows/ubuntu-system-ci.yml @@ -0,0 +1,104 @@ +# Runs the installation tests and valgrind tests in the following cases: +# 1) every day at 3 am +# 2) a pull request to the master branch + +name: Ubuntu System CI + +on: + schedule: + # Every day at 3 am + - cron: '0 3 * * *' + pull_request: + # Upon pull requests + branches: + - master + +jobs: + build: + strategy: + matrix: + os: [ubuntu-18.04, ubuntu-20.04] + runs-on: ${{ matrix.os }} + + steps: + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y libopenmpi-dev libomp5 libomp-dev libnuma-dev freeglut3-dev \ + libreadline-dev libsqlite3-dev tk-dev python-openssl ninja-build \ + valgrind + curl -L -O https://github.com/Kitware/CMake/releases/download/v3.17.3/cmake-3.17.3-Linux-x86_64.sh + chmod +x cmake-3.17.3-Linux-x86_64.sh + sudo ./cmake-3.17.3-Linux-x86_64.sh --skip-license --prefix=/usr/local + + - name: Set OSVERS + run: echo "::set-env name=OSVERS::$(lsb_release -si)-$(lsb_release -sr)" + + - name: Cache pyenv + uses: actions/cache@v2 + id: pyenv-cache + with: + path: ~/.pyenv + key: ${{ env.OSVERS }}-pyenv-3.6.9 + + - name: Install pyenv dependency + if: steps.pyenv-cache.outputs.cache-hit != 'true' + run: | + curl https://pyenv.run | bash + export PATH="$HOME/.pyenv/bin:$PATH" + eval "$(pyenv init -)" + PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install 3.6.9 + pyenv shell 3.6.9 + + - name: Checkout BioDynaMo + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Cache Third Party Packages + uses: actions/cache@v2 + with: + path: build/third_party + key: ${{ env.OSVERS }}-third-party-${{ hashFiles('cmake/external/SHA256Digests.cmake') }} + + - name: Build BioDynaMo + shell: bash + run: | + cmake -G Ninja \ + -Dparaview=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -B build + cmake --build build --parallel --config Release + + - name: Valgrind tests BioDynaMo + shell: bash + working-directory: build + run: | + . bin/thisbdm.sh + export DISPLAY=:99.0 + ../util/xvfb-initd.sh start + ninja run-valgrind + ../util/xvfb-initd.sh stop + + - name: System tests BioDynaMo + shell: bash + working-directory: build + run: | + . bin/thisbdm.sh + sudo git config --system user.name "Test User" + sudo git config --system user.email user@test.com + export DISPLAY=:99.0 + ../util/xvfb-initd.sh start + ninja run-demos + ../util/xvfb-initd.sh stop + + - name: Notify Slack + uses: 8398a7/action-slack@v3 + with: + status: ${{ job.status }} + author_name: Integration Test + fields: repo,ref,message,author,action,workflow # default: repo,commit + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # optional + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} # required + if: always() # Pick up events even if the job fails or is canceled. diff --git a/cmake/Website.cmake b/cmake/Website.cmake index 4b98bf0f4..54d68e9fa 100644 --- a/cmake/Website.cmake +++ b/cmake/Website.cmake @@ -19,6 +19,7 @@ function(GenerateStaticFiles TARGET FLAGS) file(MAKE_DIRECTORY "${WEB_DIR}") add_custom_target(${TARGET} WORKING_DIRECTORY "${WEB_DIR}" + COMMAND rm -rf ${WEB_DIR} && mkdir ${WEB_DIR} COMMAND git clone https://github.com/BioDynaMo/website.git . # TODO: merge into master to avoid this COMMAND git checkout web-updates From b86242ca26044b8ec9c76a229cca92fc92a5c711 Mon Sep 17 00:00:00 2001 From: Ahmad Hesam Date: Thu, 18 Jun 2020 16:30:48 +0200 Subject: [PATCH 39/39] Add 3s of sleep after running xvfb Thould reduce failures due to bad display server (coming from paraview tests) --- .github/workflows/centos-ci.yml | 1 + .github/workflows/centos-system-ci.yml | 2 ++ .github/workflows/ubuntu-ci.yml | 1 + .github/workflows/ubuntu-system-ci.yml | 2 ++ 4 files changed, 6 insertions(+) diff --git a/.github/workflows/centos-ci.yml b/.github/workflows/centos-ci.yml index 592789059..2d728cd07 100644 --- a/.github/workflows/centos-ci.yml +++ b/.github/workflows/centos-ci.yml @@ -82,6 +82,7 @@ jobs: . bin/thisbdm.sh export DISPLAY=:99.0 ../util/xvfb-initd.sh start + sleep 3 ninja run-unit-tests ../util/xvfb-initd.sh stop diff --git a/.github/workflows/centos-system-ci.yml b/.github/workflows/centos-system-ci.yml index 2a627cdbc..d58b78a13 100644 --- a/.github/workflows/centos-system-ci.yml +++ b/.github/workflows/centos-system-ci.yml @@ -89,6 +89,7 @@ jobs: . bin/thisbdm.sh export DISPLAY=:99.0 ../util/xvfb-initd.sh start + sleep 3 ninja run-valgrind ../util/xvfb-initd.sh stop @@ -104,6 +105,7 @@ jobs: git config --system user.email user@test.com export DISPLAY=:99.0 ../util/xvfb-initd.sh start + sleep 3 ninja run-demos ../util/xvfb-initd.sh stop diff --git a/.github/workflows/ubuntu-ci.yml b/.github/workflows/ubuntu-ci.yml index a076b2553..e5951c585 100644 --- a/.github/workflows/ubuntu-ci.yml +++ b/.github/workflows/ubuntu-ci.yml @@ -69,6 +69,7 @@ jobs: . bin/thisbdm.sh export DISPLAY=:99.0 ../util/xvfb-initd.sh start + sleep 3 ninja run-unit-tests ../util/xvfb-initd.sh stop diff --git a/.github/workflows/ubuntu-system-ci.yml b/.github/workflows/ubuntu-system-ci.yml index b418c20e5..dbbb4ca59 100644 --- a/.github/workflows/ubuntu-system-ci.yml +++ b/.github/workflows/ubuntu-system-ci.yml @@ -77,6 +77,7 @@ jobs: . bin/thisbdm.sh export DISPLAY=:99.0 ../util/xvfb-initd.sh start + sleep 3 ninja run-valgrind ../util/xvfb-initd.sh stop @@ -89,6 +90,7 @@ jobs: sudo git config --system user.email user@test.com export DISPLAY=:99.0 ../util/xvfb-initd.sh start + sleep 3 ninja run-demos ../util/xvfb-initd.sh stop