Interface to enable MFEM-based EM solves in MOOSE

.github/workflows/main.yml

@@ -3,9 +3,11 @@ on:
   push:
     branches:
        - master
+       - alexanderianblair/mfem-moose
   pull_request:
     branches:
        - master
+       - alexanderianblair/mfem-moose
 
 jobs:
   docker-build-test-ubuntu:

.gitignore

@@ -242,4 +242,7 @@ conf_vars.mk
 *.prof
 
 # Tarred Docker images
-*docker/**/*.tar.gz
+*docker/**/*.tar.gz
+
+# MFEM Joule outputs
+*Joule_000*

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "contrib/hephaestus"]
+	path = contrib/hephaestus
+	url = https://github.com/aurora-multiphysics/hephaestus

Makefile

@@ -47,12 +47,21 @@ XFEM                        := no
 include $(MOOSE_DIR)/modules/modules.mk
 ###############################################################################
 
+.PHONY : printer
+
+printer:
+	@echo $(MFEM_LIB)
+	@echo $(MFEM_INC)
+
 # dep apps
 APPLICATION_DIR    := $(CURDIR)
+APOLLO_DIR         := $(CURDIR)
 APPLICATION_NAME   := apollo
 BUILD_EXEC         := yes
 GEN_REVISION       := no
+DEP_APPS           := $(shell $(FRAMEWORK_DIR)/scripts/find_dep_apps.py $(APPLICATION_NAME))
 include            $(FRAMEWORK_DIR)/app.mk
 
+
 ###############################################################################
 # Additional special case targets should be added here

apollo.mk

@@ -0,0 +1,10 @@
+#
+# Check the existence of the contrib submodules and build accordingly
+#
+MFEM_DIR?=$(APPLICATION_DIR)/../mfem/build
+include $(MFEM_DIR)/config/config.mk
+
+
+HEPHAESTUS_DIR?=$(APPLICATION_DIR)/contrib/hephaestus
+ADDITIONAL_CPPFLAGS += -DHEPHAESTUS_ENABLED
+include $(APPLICATION_DIR)/contrib/hephaestus.mk

contrib/hephaestus.mk

@@ -0,0 +1,17 @@
+############################### HEPHAESTUS #######################################h
+###############################################################################
+# source files
+
+MFEM_INC= -I$(MFEM_DIR)  -I$(MFEM_SOURCE_DIR) -I$(MFEM_SOURCE_DIR)/miniapps/common
+MFEM_LIB= -L$(MFEM_DIR) -lmfem -lrt -L$(MFEM_DIR)/miniapps/common -lmfem-common
+
+HEPHAESTUS_INCDIR = $(HEPHAESTUS_DIR)/src/hephaestus_lib
+HEPHAESTUS_LIBDIR = $(HEPHAESTUS_DIR)/lib
+HEPHAESTUS_INC = -I$(HEPHAESTUS_INCDIR)
+HEPHAESTUS_LIB = -Wl,-rpath,$(HEPHAESTUS_LIBDIR) -L$(HEPHAESTUS_LIBDIR) -lhephaestus -lnetcdf
+
+ADDITIONAL_INCLUDES += $(HEPHAESTUS_INC) $(MFEM_INC) -I$(MFEM_INC_DIR)/config
+ADDITIONAL_LIBS += $(HEPHAESTUS_LIB) $(MFEM_LIB)
+
+$(info ADDITIONAL_INCLUDES = $(ADDITIONAL_INCLUDES));
+$(info ADDITIONAL_LIBS = $(ADDITIONAL_LIBS));

docker/apollo-ubuntu/Dockerfile

@@ -5,22 +5,100 @@
 # docker build -t apollo-ubuntu --build-arg build_git_sha=${GITHUB_SHA} .
 
 # Get MOOSE image
-FROM helenbrooks/moose-ubuntu
+FROM herter4171/ubuntu-moose:latest
 
 # By default one core is used to compile
 ARG compile_cores=1
 
-# By default checkout main branch
-ARG build_git_sha="master"
+# By default checkout mfem-moose branch
+ARG WORKDIR="opt"
+ARG build_git_sha="alexanderianblair/mfem-moose"
+ARG hypre_git_sha="562c29a"
 
-RUN cd /home && \
-cd moose/framework/src/base && \
-sed -i 's/(type.family == NEDELEC_ONE)/(type.family == NEDELEC_ONE||type.family == LAGRANGE_VEC)/g' Assembly.C
+# Install apt dependencies
+RUN apt-get update && apt-get install -y \
+    clang-format-9 \
+    curl \
+    libboost-all-dev \
+    libnetcdf-dev \
+    libssl-dev \
+    xxd \
+    libfontconfig1-dev \
+    libfreetype6-dev \
+    libsdl2-dev \
+    libglew-dev \
+    libglm-dev \
+    libpng-dev \
+    ninja-build
 
-RUN cd /home && \
+# Enable Nedelec element curl calculations in MOOSE
+RUN cd /$WORKDIR && \
+    cd moose/framework/src/base && \
+    sed -i 's/(type.family == NEDELEC_ONE)/(type.family == NEDELEC_ONE||type.family == LAGRANGE_VEC)/g' Assembly.C
+
+# Install CMake
+RUN cd /$WORKDIR && \
+    git clone https://github.com/Kitware/CMake.git && \
+    cd CMake/ && \
+    git checkout 78ae759 && \
+    ./bootstrap && \
+    make && \
+    make install && \
+    alias cmake='/$WORKDIR/CMake/bin/cmake'
+
+# Install METIS 5.1.0
+RUN cd /$WORKDIR && \
+    curl -O http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/metis-5.1.0.tar.gz && \
+    tar -zxvf metis-* && \
+    cd metis-5.1.0 && \
+    make config shared=1 && \
+    make && \
+    make install && \
+    ln -s ../build/Linux-x86_64/libmetis/libmetis.so lib
+
+# Install MUMPS
+RUN cd /$WORKDIR && \
+    git clone https://github.com/scivision/mumps && \
+    cd mumps/ && \
+    mkdir build && \
+    cd build && \
+    cmake --preset=build -DBUILD_SHARED_LIBS=on -Dscotch=true -Dintsize64=false -Darith="s;d;c;z" -Dautobuild=true  \
+    -DCMAKE_INSTALL_PREFIX=/usr/local/ .. && \
+    make && \
+    make install
+
+# Build MFEM and common miniapp
+RUN cd /$WORKDIR && \
+    git clone https://github.com/mfem/mfem.git && \
+    cd mfem && \
+    mkdir build && \
+    cd build && \
+    cmake .. -DCMAKE_BUILD_TYPE=Debug -DMFEM_USE_MPI=YES -DMFEM_USE_METIS_5=YES -DMETIS_DIR=../../metis-5.1.0 \
+    -DMFEM_USE_OPENMP=YES -DMFEM_THREAD_SAFE=NO -DMFEM_SHARED=YES -DCMAKE_CXX_FLAGS_DEBUG=-fPIC -DMFEM_USE_NETCDF=YES \
+    -DCMAKE_POLICY_DEFAULT_CMP0057=NEW  &&\
+    make && \
+    cd miniapps/common && \
+    make
+
+# Build GLVis
+RUN cd /$WORKDIR && \
+    git clone https://github.com/GLVis/glvis.git && \
+    cd glvis/ && \
+    mkdir build && \
+    cd build/ && \
+    cmake -G Ninja -DMFEM_DIR=/opt/mfem/build .. && \
+    ninja
+
+# Build Apollo with Hephaestus
+RUN cd /$WORKDIR && \
     git clone https://github.com/aurora-multiphysics/apollo.git && \
     cd apollo && \
     git checkout "$build_git_sha" && \
-    make -j"$compile_cores" && \
-    make -j"$compile_cores" test &&\
-    ./run_tests
+    git submodule update --init --recursive && \
+    cd contrib/hephaestus/ && \
+    mkdir build && \
+    cd build && \
+    cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE=Debug -DMFEM_DIR=/$WORKDIR/mfem/build -DMFEM_COMMON_INCLUDES=/$WORKDIR/mfem/miniapps/common  .. && \
+    make && \
+    cd /$WORKDIR/apollo && \
+    make -j"$compile_cores"

include/bcs/MFEMBoundaryCondition.h

@@ -0,0 +1,23 @@
+#pragma once
+
+#include "GeneralUserObject.h"
+#include "boundary_conditions.hpp"
+
+class MFEMBoundaryCondition : public GeneralUserObject
+{
+public:
+  static InputParameters validParams();
+
+  MFEMBoundaryCondition(const InputParameters & parameters);
+  virtual ~MFEMBoundaryCondition();
+
+  virtual hephaestus::BoundaryCondition * getBC();
+  virtual void execute() override {}
+  virtual void initialize() override {}
+  virtual void finalize() override {}
+
+protected:
+  std::vector<BoundaryName> _boundary_names;
+  mfem::Array<int> bdr_attr;
+  hephaestus::BoundaryCondition * _boundary_condition;
+};

include/bcs/MFEMFunctionDirichletBC.h

@@ -0,0 +1,24 @@
+#pragma once
+
+#include "MFEMBoundaryCondition.h"
+#include "boundary_conditions.hpp"
+
+libMesh::Point PointFromMFEMVector(const mfem::Vector & vec);
+
+class MFEMFunctionDirichletBC : public MFEMBoundaryCondition
+{
+public:
+  static InputParameters validParams();
+
+  MFEMFunctionDirichletBC(const InputParameters & parameters);
+  virtual ~MFEMFunctionDirichletBC();
+
+  virtual hephaestus::BoundaryCondition * getBC() override;
+  virtual void execute() override {}
+  virtual void initialize() override {}
+  virtual void finalize() override {}
+
+protected:
+  const Function & _func;
+  mfem::FunctionCoefficient _function_coeff;
+};

include/kernels/CoupledVectorForce.h

@@ -2,12 +2,6 @@
 
 #include "VectorKernel.h"
 
-// Forward Declaration
-class CoupledVectorForce;
-
-template <>
-InputParameters validParams<CoupledVectorForce>();
-
 /**
  * Class to use a coupled auxvariable to apply a vector body force.
  */

include/kernels/MatTimeDerivative.h

@@ -12,11 +12,6 @@
 #include "TimeDerivative.h"
 #include "MaterialProperty.h"
 
-class MatTimeDerivative;
-
-template <>
-InputParameters validParams<MatTimeDerivative>();
-
 /**
  * Time derivative term multiplied by a material coefficient
  */
@@ -34,4 +29,3 @@ class MatTimeDerivative : public TimeDerivative
   /// The coefficient the time derivative is multiplied with
   const MaterialProperty<Real> & _sigma;
 };
-

include/materials/MFEMConductor.h

@@ -0,0 +1,35 @@
+#pragma once
+
+#include "MFEMMaterial.h"
+#include "materials.hpp"
+
+class MFEMConductor : public MFEMMaterial
+{
+public:
+  static InputParameters validParams();
+
+  MFEMConductor(const InputParameters & parameters);
+  virtual ~MFEMConductor();
+
+  /// EM constants
+  static const Real _mu0;
+  static const Real _epsilon0;
+
+protected:
+  const Real & _input_electrical_conductivity;
+  const Real & _input_rel_permittivity;
+  const Real & _input_rel_permeability;
+  const Real & _input_thermal_conductivity;
+  const Real & _input_heat_capacity;
+
+  mfem::ConstantCoefficient _electrical_conductivity_coeff;
+  mfem::ConstantCoefficient _permittivity_coeff;
+  mfem::ConstantCoefficient _permeability_coeff;
+  mfem::ConstantCoefficient _resistivity_coeff;
+  mfem::ConstantCoefficient _reluctivity_coeff;
+
+  mfem::ConstantCoefficient _heat_capacity_coeff;
+  mfem::ConstantCoefficient _inverse_heat_capacity_coeff;
+  mfem::ConstantCoefficient _thermal_conductivity_coeff;
+  mfem::ConstantCoefficient _inverse_thermal_conductivity_coeff;
+};

include/materials/MFEMMaterial.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include "GeneralUserObject.h"
+#include "materials.hpp"
+
+class MFEMMaterial : public GeneralUserObject
+{
+public:
+  static InputParameters validParams();
+
+  MFEMMaterial(const InputParameters & parameters);
+  virtual ~MFEMMaterial();
+
+  virtual void execute() override {}
+  virtual void initialize() override {}
+  virtual void finalize() override {}
+
+  std::map<std::string, mfem::Coefficient *> scalar_property_map;
+  std::vector<SubdomainName> blocks;
+};

include/mesh/MFEMMesh.h

@@ -0,0 +1,28 @@
+//* This file is part of the MOOSE framework
+//* https://www.mooseframework.org
+//*
+//* All rights reserved, see COPYRIGHT for full restrictions
+//* https://github.com/idaholab/moose/blob/master/COPYRIGHT
+//*
+//* Licensed under LGPL 2.1, please see LICENSE for details
+//* https://www.gnu.org/licenses/lgpl-2.1.html
+
+#pragma once
+
+#include "GeneratedMesh.h"
+
+/**
+ * MFEMMesh
+ */
+class MFEMMesh : public GeneratedMesh
+{
+public:
+  static InputParameters validParams();
+
+  MFEMMesh(const InputParameters & parameters);
+  MFEMMesh(const MFEMMesh & other_mesh) = default;
+
+public:
+  /// the file_name from whence this mesh came
+  std::string file_name;
+};

include/postprocessors/BlockAveragePostprocessor.h

@@ -2,11 +2,6 @@
 
 #include "GeneralPostprocessor.h"
 #include "BlockAverageValue.h"
-// Forward Declarations
-class BlockAveragePostprocessor;
-
-template <>
-InputParameters validParams<BlockAveragePostprocessor>();
 
 /**
  *

include/postprocessors/ElementNedelecL2Error.h

@@ -4,12 +4,6 @@
 
 class Function;
 
-// Forward Declarations
-class ElementNedelecL2Error;
-
-template <>
-InputParameters validParams<ElementNedelecL2Error>();
-
 class ElementNedelecL2Error : public ElementIntegralPostprocessor
 {
 public: