Merge pull request #32 from aurora-multiphysics/EdwardPalmer99/Distri…

…buted-Mesh-Transfer-V1

Enable transfer of distributed MOOSE mesh (v1: distributed MOOSE --> MFEMMesh --> MFEMParMesh)

include/mesh/CoupledMFEMMesh.h

@@ -98,6 +98,11 @@ class CoupledMFEMMesh : public ExclusiveMFEMMesh
    */
   std::vector<int> getSideBoundaryIDs() const;
 
+  /**
+   * Returns a pointer to the first element on the processor.
+   */
+  const Elem * getFirstElementOnProcessor() const;
+
   /**
    * Override methods in Exclusive MFEMMesh.
    */

src/mesh/CoupledMFEMMesh.C

@@ -98,10 +98,9 @@ CoupledMFEMMesh::buildBoundaryInfo(std::map<int, std::vector<int>> & element_ids
 void
 CoupledMFEMMesh::buildLibmesh2DElementInfo()
 {
-  // TODO: - this will not work with distributed. Can we just get the first local element?
-  const Elem * element_ptr = elemPtr(0);
+  auto first_element_ptr = getFirstElementOnProcessor();
 
-  _num_nodes_per_element = element_ptr->n_nodes();
+  _num_nodes_per_element = first_element_ptr->n_nodes();
 
   switch (_num_nodes_per_element)
   {
@@ -144,10 +143,9 @@ CoupledMFEMMesh::buildLibmesh2DElementInfo()
 void
 CoupledMFEMMesh::buildLibmesh3DElementInfo()
 {
-  // TODO: - this will not work with distributed. Can we just get the first local element?
-  const Elem * element_ptr = elemPtr(0);
+  auto first_element_ptr = getFirstElementOnProcessor();
 
-  _num_nodes_per_element = element_ptr->n_nodes();
+  _num_nodes_per_element = first_element_ptr->n_nodes();
 
   switch (_num_nodes_per_element)
   {
@@ -278,6 +276,28 @@ CoupledMFEMMesh::getSideBoundaryIDs() const
   return side_boundary_ids;
 }
 
+const Elem *
+CoupledMFEMMesh::getFirstElementOnProcessor() const
+{
+  Elem * first_element_ptr = nullptr;
+
+  auto local_elements_begin = getMesh().local_elements_begin();
+  auto local_elements_end = getMesh().local_elements_end();
+
+  for (auto iterator = local_elements_begin; iterator != local_elements_end; iterator++)
+  {
+    first_element_ptr = *iterator;
+    break;
+  }
+
+  if (!first_element_ptr)
+  {
+    mooseError("Unable to get the first element on processor ", getMesh().processor_id());
+  }
+
+  return first_element_ptr;
+}
+
 void
 CoupledMFEMMesh::buildLibmeshElementAndFaceInfo()
 {
@@ -290,9 +310,9 @@ CoupledMFEMMesh::getLibmeshBlockIDs() const
 {
   auto & libmesh = getMesh();
 
-  // TODO: - may not work with distributed meshes.
+  // Identify all subdomains (blocks) in the entire mesh (global == true).
   std::set<subdomain_id_type> block_ids_set;
-  libmesh.subdomain_ids(block_ids_set);
+  libmesh.subdomain_ids(block_ids_set, true);
 
   std::vector<int> unique_block_ids(block_ids_set.size());
 
@@ -314,8 +334,11 @@ CoupledMFEMMesh::buildElementAndNodeIDs(const std::vector<int> & unique_block_id
   {
     std::vector<int> elements_in_block;
 
-    for (auto element_iterator = getMesh().active_subdomain_elements_begin(block_id);
-         element_iterator != getMesh().active_subdomain_elements_end(block_id);
+    auto active_block_elements_begin = getMesh().active_subdomain_elements_begin(block_id);
+    auto active_block_elements_end = getMesh().active_subdomain_elements_end(block_id);
+
+    for (auto element_iterator = active_block_elements_begin;
+         element_iterator != active_block_elements_end;
          element_iterator++)
     {
       auto element_ptr = *element_iterator;
@@ -377,6 +400,16 @@ CoupledMFEMMesh::buildUniqueCornerNodeIDs(
 void
 CoupledMFEMMesh::buildMFEMMesh()
 {
+  // If the mesh is distributed and split between more than one processor,
+  // we need to call allgather on each processor. This will gather the nodes
+  // and elements onto each processor.
+  const bool is_distributed = !getMesh().is_replicated();
+
+  if (is_distributed && getMesh().n_processors() > 1)
+  {
+    getMesh().allgather();
+  }
+
   // 1. Retrieve information about the elements used within the mesh.
   buildLibmeshElementAndFaceInfo();
 

test/tests/unit/distributed_transfers/elemental_var_coupled.i

@@ -0,0 +1,99 @@
+[Mesh]
+  type = CoupledMFEMMesh
+  file = gold/cylinder-hex-q2.cpr
+  dim = 3
+  parallel_type=distributed
+[]
+
+[Problem]
+  type = MFEMProblem
+[]
+
+[Formulation]
+  type = EBFormulation
+  e_field_name = electric_field
+  b_field_name = magnetic_flux_density
+  magnetic_reluctivity_name = magnetic_reluctivity
+  magnetic_permeability_name = magnetic_permeability
+  electric_conductivity_name = electrical_conductivity
+[]
+
+[AuxVariables]
+  [./dummy_mfem]
+    type = MFEMVariable
+    fespace_name = HCurlFESpace
+    fespace_type = ND
+    order = SECOND
+  [../]
+  [./dummy_moose_elemental]
+    family = MONOMIAL
+    order = CONSTANT
+    [./InitialCondition]
+      type = FunctionIC
+      function = 2-x
+    [../]    
+  [../]  
+[../]
+
+[Materials]
+  [./copper]
+    type = MFEMConductor
+    electrical_conductivity_coeff = CopperEConductivity
+    electric_permittivity_coeff = CopperPermittivity
+    magnetic_permeability_coeff = CopperPermeability
+    block = 1
+  [../]
+  [./air]
+    type = MFEMConductor
+    electrical_conductivity_coeff = AirEConductivity
+    electric_permittivity_coeff = AirPermittivity
+    magnetic_permeability_coeff = AirPermeability
+    block = 2
+  [../]
+[]
+
+[Coefficients]
+  [./CopperEConductivity]
+    type = MFEMConstantCoefficient
+    value = 5.96e7
+  [../]
+  [./CopperPermeability]
+    type = MFEMConstantCoefficient
+    value = 1.25663706e-6
+  [../]  
+  [./CopperPermittivity]
+    type = MFEMConstantCoefficient
+    value = 0.0
+  [../]
+
+  [./AirEConductivity]
+    type = MFEMConstantCoefficient
+    value = 1.0
+  [../]
+  [./AirPermeability]
+    type = MFEMConstantCoefficient
+    value = 1.25663706e-6
+  [../]
+  [./AirPermittivity]
+    type = MFEMConstantCoefficient
+    value = 0.0
+  [../]
+[]
+
+[Executioner]
+  type = Transient
+  dt = 0.5
+  start_time = 0.0
+  end_time = 0.5
+  
+  l_tol = 1e-16
+  l_max_its = 1000  
+[]
+
+[Outputs]
+  exodus=true
+  [VisItDataCollection]
+    type = MFEMVisItDataCollection
+    file_base = OutputData/ElementalCoupledMFEMVarTest
+  []
+[]

test/tests/unit/distributed_transfers/tests

@@ -0,0 +1,11 @@
+[Tests]
+  design = 'syntax/Problem/index.md'
+  [./CoupledElementalMFEMVar]
+    type = 'Exodiff'
+    input = 'elemental_var_coupled.i'
+    exodiff = 'elemental_var_coupled_out.e'  
+    max_parallel = 2
+    min_parallel = 2
+    requirement = 'Apollo shall have the transfer from distributed MOOSE meshes'
+  []  
+[]