From 83ecf62e0b4363e1fa8dd83f3e68e9046d329496 Mon Sep 17 00:00:00 2001 From: Christos Tsolakis <6725596+ChristosT@users.noreply.github.com> Date: Tue, 30 Jul 2024 23:40:08 +0200 Subject: [PATCH] Catalyst support (#4011) ## Summary This PR completes the work of @andrewcombs and @c-wetterer-nelson from #3444 by adding a docker image that includes ParaView and can be used for testing. It also adds a minor fix a02c1a5110ce6104a16aba4eee4685c9196eb758 discovered when testing these changes with the current master of WarpX. ## Additional background Supersedes and closes #3444 ## Checklist The proposed changes: - [x] fix a bug or incorrect behavior in AMReX - [x] add new capabilities to AMReX - [ ] changes answers in the test suite to more than roundoff level - [ ] are likely to significantly affect the results of downstream AMReX users - [ ] include documentation in the code and/or rst files, if appropriate --------- Co-authored-by: Andrew Combs Co-authored-by: Corey Wetterer-Nelson --- .github/workflows/catalyst.yml | 33 +++++++++++++++++++ .../source/BuildingAMReX.rst | 2 ++ Src/CMakeLists.txt | 3 +- .../AMReX_Conduit_Blueprint_ParticlesI.H | 33 +++++++++++-------- Src/Extern/Conduit/CMakeLists.txt | 1 + Src/Extern/Conduit/Make.package | 2 +- Tools/CMake/AMReXConfig.cmake.in | 6 ++++ Tools/CMake/AMReXOptions.cmake | 5 +++ Tools/CMake/AMReXSetDefines.cmake | 3 ++ Tools/CMake/AMReXThirdPartyLibraries.cmake | 9 +++++ Tools/CMake/AMReX_Config_ND.H.in | 1 + Tools/GNUMake/Make.defs | 11 +++++++ Tools/GNUMake/packages/Make.catalyst | 16 +++++++++ 13 files changed, 108 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/catalyst.yml create mode 100644 Tools/GNUMake/packages/Make.catalyst diff --git a/.github/workflows/catalyst.yml b/.github/workflows/catalyst.yml new file mode 100644 index 00000000000..f6940574737 --- /dev/null +++ b/.github/workflows/catalyst.yml @@ -0,0 +1,33 @@ +name: 🐧 Catalyst + +on: [push, pull_request] + +concurrency: + group: ${{ github.ref }}-${{ github.head_ref }}-insituvis-catalyst + cancel-in-progress: true + +jobs: + catalyst: + name: Catalyst + runs-on: ubuntu-22.04 + if: github.event.pull_request.draft == false + env: + CXX: g++ + CC: gcc + CMAKE_PREFIX: "/opt/conduit;/opt/catalyst" + container: + image: kitware/paraview:ci-catalyst-amrex-warpx-20240701 + steps: + - uses: actions/checkout@v4 + - name: Configure + run: | + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Debug \ + -DAMReX_ENABLE_TESTS=ON \ + -DAMReX_FORTRAN=OFF \ + -DAMReX_CATALYST=ON \ + -DAMReX_CONDUIT=ON + - name: Build + run: | + cmake --build build -j 4 + diff --git a/Docs/sphinx_documentation/source/BuildingAMReX.rst b/Docs/sphinx_documentation/source/BuildingAMReX.rst index 3176730320e..90fb4d6eb30 100644 --- a/Docs/sphinx_documentation/source/BuildingAMReX.rst +++ b/Docs/sphinx_documentation/source/BuildingAMReX.rst @@ -508,6 +508,8 @@ The list of available options is reported in the :ref:`table ` bel +------------------------------+-------------------------------------------------+-------------------------+-----------------------+ | AMReX_CONDUIT | Enable Conduit support | NO | YES, NO | +------------------------------+-------------------------------------------------+-------------------------+-----------------------+ + | AMReX_CATALYST | Enable Catalyst support | NO | YES, NO | + +------------------------------+-------------------------------------------------+-------------------------+-----------------------+ | AMReX_ASCENT | Enable Ascent support | NO | YES, NO | +------------------------------+-------------------------------------------------+-------------------------+-----------------------+ | AMReX_HYPRE | Enable HYPRE interfaces | NO | YES, NO | diff --git a/Src/CMakeLists.txt b/Src/CMakeLists.txt index 56cd426efb9..6e8af043e0d 100644 --- a/Src/CMakeLists.txt +++ b/Src/CMakeLists.txt @@ -43,7 +43,6 @@ include(AMReXParallelBackends) # Add definitions # include(AMReXSetDefines) - # # Find and link third party libraries if needed # @@ -188,7 +187,7 @@ if (AMReX_SENSEI) add_subdirectory(Extern/SENSEI) endif () -if (AMReX_CONDUIT OR AMReX_ASCENT) +if (AMReX_CONDUIT OR AMReX_ASCENT OR AMReX_CATALYST) add_subdirectory(Extern/Conduit) endif () diff --git a/Src/Extern/Conduit/AMReX_Conduit_Blueprint_ParticlesI.H b/Src/Extern/Conduit/AMReX_Conduit_Blueprint_ParticlesI.H index e4186ba247b..b3275c391e4 100644 --- a/Src/Extern/Conduit/AMReX_Conduit_Blueprint_ParticlesI.H +++ b/Src/Extern/Conduit/AMReX_Conduit_Blueprint_ParticlesI.H @@ -172,25 +172,30 @@ ParticleTileToBlueprint(const ParticleTile(&soa.GetIntData(0)[0]), - num_particles); + // for soa entries, we can use standard strides, + // since these are contiguous arrays - // cpu is the second int entry - conduit::Node &n_f_cpu = n_fields[topology_name + "_cpu"]; + // id is the first int entry + conduit::Node &n_f_id = n_fields[topology_name + "_id"]; - n_f_cpu["topology"] = topology_name; - n_f_cpu["association"] = "element"; - n_f_cpu["values"].set_external(const_cast(&soa.GetIntData(0)[0]), - num_particles); + n_f_id["topology"] = topology_name; + n_f_id["association"] = "element"; + n_f_id["values"].set_external(const_cast(&soa.GetIntData(0)[0]), + num_particles); + // cpu is the second int entry + conduit::Node &n_f_cpu = n_fields[topology_name + "_cpu"]; + + n_f_cpu["topology"] = topology_name; + n_f_cpu["association"] = "element"; + n_f_cpu["values"].set_external(const_cast(&soa.GetIntData(0)[0]), + num_particles); + + } } // -------------------------------- diff --git a/Src/Extern/Conduit/CMakeLists.txt b/Src/Extern/Conduit/CMakeLists.txt index 75e84099f1d..07c4722ac4b 100644 --- a/Src/Extern/Conduit/CMakeLists.txt +++ b/Src/Extern/Conduit/CMakeLists.txt @@ -4,6 +4,7 @@ foreach(D IN LISTS AMReX_SPACEDIM) # TODO: Particles PR merges another file target_sources(amrex_${D}d PRIVATE + AMReX_Conduit_Blueprint_ParticlesI.H AMReX_Conduit_Blueprint.H AMReX_Conduit_Blueprint.cpp ) diff --git a/Src/Extern/Conduit/Make.package b/Src/Extern/Conduit/Make.package index 2fc13b7d606..def60ba1a74 100644 --- a/Src/Extern/Conduit/Make.package +++ b/Src/Extern/Conduit/Make.package @@ -3,7 +3,7 @@ # CEXE_sources += AMReX_Conduit_Blueprint.cpp - +CEXE_headers += AMReX_Conduit_Blueprint_ParticlesI.H CEXE_headers += AMReX_Conduit_Blueprint.H VPATH_LOCATIONS += $(AMREX_HOME)/Src/Extern/Conduit diff --git a/Tools/CMake/AMReXConfig.cmake.in b/Tools/CMake/AMReXConfig.cmake.in index 7731e988e68..f5045b715cb 100644 --- a/Tools/CMake/AMReXConfig.cmake.in +++ b/Tools/CMake/AMReXConfig.cmake.in @@ -79,6 +79,7 @@ set(AMReX_PARTICLES_FOUND @AMReX_PARTICLES@) set(AMReX_P@AMReX_PARTICLES_PRECISION@_FOUND ON) set(AMReX_SENSEI_FOUND @AMReX_SENSEI@) set(AMReX_CONDUIT_FOUND @AMReX_CONDUIT@) +set(AMReX_CATALYST_FOUND @AMReX_CATALYST@) set(AMReX_ASCENT_FOUND @AMReX_ASCENT@) set(AMReX_HYPRE_FOUND @AMReX_HYPRE@) set(AMReX_PETSC_FOUND @AMReX_PETSC@) @@ -134,6 +135,7 @@ set(AMReX_PARTICLES_PRECISION @AMReX_PARTICLES_PRECISION@) set(AMReX_SENSEI @AMReX_SENSEI@) set(AMReX_NO_SENSEI_AMR_INST @AMReX_NO_SENSEI_AMR_INST@) set(AMReX_CONDUIT @AMReX_CONDUIT@) +set(AMReX_CATALYST @AMReX_CATALYST@) set(AMReX_ASCENT @AMReX_ASCENT@) set(AMReX_HYPRE @AMReX_HYPRE@) set(AMReX_PETSC @AMReX_PETSC@) @@ -202,6 +204,10 @@ if (@AMReX_ASCENT@) find_dependency(Ascent REQUIRED) endif () +if (@AMReX_CATALYST@) + find_dependency(Catalyst REQUIRED) +endif () + if (@AMReX_CONDUIT@) find_dependency(Conduit REQUIRED) endif () diff --git a/Tools/CMake/AMReXOptions.cmake b/Tools/CMake/AMReXOptions.cmake index a59456d7bb8..3e5d4c8bdb4 100644 --- a/Tools/CMake/AMReXOptions.cmake +++ b/Tools/CMake/AMReXOptions.cmake @@ -320,6 +320,11 @@ print_option( AMReX_NO_SENSEI_AMR_INST ) option( AMReX_CONDUIT "Enable Conduit support" OFF ) print_option( AMReX_CONDUIT ) +# Catalyst +cmake_dependent_option( AMReX_CATALYST "Enable Catalyst support" OFF + "AMReX_CONDUIT" OFF ) +print_option( AMReX_CATALYST ) + # Ascent cmake_dependent_option( AMReX_ASCENT "Enable Ascent support" OFF "AMReX_CONDUIT" OFF ) diff --git a/Tools/CMake/AMReXSetDefines.cmake b/Tools/CMake/AMReXSetDefines.cmake index 783b1ab83d6..d271321bde4 100644 --- a/Tools/CMake/AMReXSetDefines.cmake +++ b/Tools/CMake/AMReXSetDefines.cmake @@ -148,6 +148,9 @@ add_amrex_define( AMREX_NO_SENSEI_AMR_INST NO_LEGACY IF AMReX_NO_SENSEI_AMR_INST # Conduit Support add_amrex_define( AMREX_USE_CONDUIT NO_LEGACY IF AMReX_CONDUIT ) +# Catalyst Support +add_amrex_define( AMREX_USE_CATALYST NO_LEGACY IF AMReX_CATALYST ) + # Ascent Support add_amrex_define( AMREX_USE_ASCENT NO_LEGACY IF AMReX_ASCENT ) diff --git a/Tools/CMake/AMReXThirdPartyLibraries.cmake b/Tools/CMake/AMReXThirdPartyLibraries.cmake index f8f49e9c478..abe62a2ebc9 100644 --- a/Tools/CMake/AMReXThirdPartyLibraries.cmake +++ b/Tools/CMake/AMReXThirdPartyLibraries.cmake @@ -90,6 +90,15 @@ if (AMReX_ASCENT) # Ascent will find conduit, so check for Ascent first endforeach() endif () +# +# Catalyst +# +if (AMReX_CATALYST) + find_package(Catalyst REQUIRED PATHS "$ENV{CATALYST_IMPLEMENTATION_PATHS}") + foreach(D IN LISTS AMReX_SPACEDIM) + target_link_libraries(amrex_${D}d PUBLIC catalyst::catalyst) + endforeach() +endif () # # Conduit diff --git a/Tools/CMake/AMReX_Config_ND.H.in b/Tools/CMake/AMReX_Config_ND.H.in index 07df0fb53cb..3296a403ff0 100644 --- a/Tools/CMake/AMReX_Config_ND.H.in +++ b/Tools/CMake/AMReX_Config_ND.H.in @@ -42,6 +42,7 @@ #cmakedefine AMREX_USE_SENSEI_INSITU #cmakedefine AMREX_NO_SENSEI_AMR_INST #cmakedefine AMREX_USE_CONDUIT +#cmakedefine AMREX_USE_CATALYST #cmakedefine AMREX_USE_ASCENT #cmakedefine AMREX_USE_EB #cmakedefine AMREX_USE_CUDA diff --git a/Tools/GNUMake/Make.defs b/Tools/GNUMake/Make.defs index a1a2aa105b0..6a1c51c9292 100644 --- a/Tools/GNUMake/Make.defs +++ b/Tools/GNUMake/Make.defs @@ -216,6 +216,12 @@ else USE_CONDUIT := FALSE endif +ifdef USE_CATALYST + USE_CATALYST := $(strip $(USE_CATALYST)) +else + USE_CATALYST := FALSE +endif + ifdef USE_ASCENT USE_ASCENT := $(strip $(USE_ASCENT)) else @@ -1057,6 +1063,11 @@ ifeq ($(USE_CONDUIT),TRUE) include $(AMREX_HOME)/Tools/GNUMake/packages/Make.conduit endif +ifeq ($(USE_CATALYST),TRUE) + $(info Loading $(AMREX_HOME)/Tools/GNUMake/packages/Make.catalyst...) + include $(AMREX_HOME)/Tools/GNUMake/packages/Make.catalyst +endif + ifeq ($(USE_ASCENT),TRUE) $(info Loading $(AMREX_HOME)/Tools/GNUMake/packages/Make.ascent...) include $(AMREX_HOME)/Tools/GNUMake/packages/Make.ascent diff --git a/Tools/GNUMake/packages/Make.catalyst b/Tools/GNUMake/packages/Make.catalyst new file mode 100644 index 00000000000..810daa906f2 --- /dev/null +++ b/Tools/GNUMake/packages/Make.catalyst @@ -0,0 +1,16 @@ +######################################################### +# Catalyst (https://gitlab.kitware.com/paraview/catalyst) Support +######################################################### + +CPPFLAGS += -DAMREX_USE_CATALYST + +ifdef CATALYST_DIR + INCLUDE_LOCATIONS += $(CATALYST_DIR)/include + VPATH_LOCATIONS += $(CATALYST_DIR)/include + LIBRARY_LOCATIONS += $(CATALYST_DIR)/lib + LIBRARIES += -Wl,-rpath,$(CATALYST_DIR)/lib + + LIBRARIES += -lcatalyst + +endif +