Fix non-blocking communication for adaptive time step

src/Hipace.H

@@ -159,7 +159,7 @@ public:
     /** Particle container for the plasma */
     PlasmaParticleContainer m_plasma_container;
     /** Number of time iterations */
-    int m_max_step = 0;
+    static int m_max_step;
     /** Time step for the beam evolution */
     static amrex::Real m_dt;
     /** Number of iterations between consecutive output dumps.

src/Hipace.cpp

@@ -23,6 +23,7 @@ namespace {
 
 Hipace* Hipace::m_instance = nullptr;
 
+int Hipace::m_max_step = 0;
 amrex::Real Hipace::m_dt = 0.0;
 bool Hipace::m_normalized_units = false;
 int Hipace::m_verbose = 0;

src/utils/AdaptiveTimeStep.cpp

@@ -25,11 +25,12 @@ void
 AdaptiveTimeStep::PassTimeStepInfo (const int nt, MPI_Comm a_comm_z)
 {
     HIPACE_PROFILE("SetTimeStep()");
-
+#ifdef AMREX_USE_MPI
     if (m_do_adaptive_time_step == 0) return;
 
     const int my_rank_z = amrex::ParallelDescriptor::MyProc();
     const int numprocs_z = amrex::ParallelDescriptor::NProcs();
+    if (numprocs_z == 0) return;
 
     if ((nt % numprocs_z == 0) && (my_rank_z >= 1))
     {
@@ -45,6 +46,7 @@ AdaptiveTimeStep::PassTimeStepInfo (const int nt, MPI_Comm a_comm_z)
                  my_rank_z-1, comm_z_tag, a_comm_z, &m_send_request);
 
     }
+#endif
 }
 
 void
@@ -78,13 +80,13 @@ AdaptiveTimeStep::Calculate (amrex::Real& dt, const int nt, MultiBeam& beams,
     // Extract properties associated with physical size of the box
     const PhysConst phys_const = get_phys_const();
 
-    if (my_rank_z == numprocs_z-1) {
-        m_timestep_data[WhichDouble::SumWeights] = 0.;
-        m_timestep_data[WhichDouble::SumWeightsTimesUz] = 0.;
-        m_timestep_data[WhichDouble::SumWeightsTimesUzSquared] = 0.;
-        m_timestep_data[WhichDouble::MinUz] = 1e30;
-        if (my_rank_z == numprocs_z -1 )
-        {
+#ifdef AMREX_USE_MPI
+    if (numprocs_z > 1) {
+        if (my_rank_z == numprocs_z-1) {
+            m_timestep_data[WhichDouble::SumWeights] = 0.;
+            m_timestep_data[WhichDouble::SumWeightsTimesUz] = 0.;
+            m_timestep_data[WhichDouble::SumWeightsTimesUzSquared] = 0.;
+            m_timestep_data[WhichDouble::MinUz] = 1e30;
             if (nt > 0){
                 // first rank receives the new dt from last rank
                 auto recv_buffer = (amrex::Real*)amrex::The_Pinned_Arena()->alloc(
@@ -98,31 +100,29 @@ AdaptiveTimeStep::Calculate (amrex::Real& dt, const int nt, MultiBeam& beams,
             }
             /* always set time step, because it will be passed */
             m_timestep_data[WhichDouble::Dt] = dt;
-
-        }
-
-    } else {
-        /* lower ranks receive all time step data from the upper rank */
-        auto recv_buffer = (amrex::Real*)amrex::The_Pinned_Arena()->alloc
-            (sizeof(amrex::Real)*WhichDouble::N);
-        MPI_Status status;
-        MPI_Recv(recv_buffer, WhichDouble::N,
-                 amrex::ParallelDescriptor::Mpi_typemap<amrex::Real>::type(),
-                 my_rank_z+1, comm_z_tag, a_comm_z, &status);
-
-        for (int idouble=0; idouble<(int) WhichDouble::N; idouble++) {
-            m_timestep_data[idouble] = recv_buffer[idouble];
+        } else {
+            /* lower ranks receive all time step data from the upper rank */
+            auto recv_buffer = (amrex::Real*)amrex::The_Pinned_Arena()->alloc
+                (sizeof(amrex::Real)*WhichDouble::N);
+            MPI_Status status;
+            MPI_Recv(recv_buffer, WhichDouble::N,
+                     amrex::ParallelDescriptor::Mpi_typemap<amrex::Real>::type(),
+                     my_rank_z+1, comm_z_tag, a_comm_z, &status);
+
+            for (int idouble=0; idouble<(int) WhichDouble::N; idouble++) {
+                m_timestep_data[idouble] = recv_buffer[idouble];
+            }
+            /* setting dt, so it can be used in the beam pusher */
+            dt = recv_buffer[WhichDouble::Dt];
+            amrex::The_Pinned_Arena()->free(recv_buffer);
         }
-        /* setting dt, so it can be used in the beam pusher */
-        dt = recv_buffer[WhichDouble::Dt];
-        amrex::The_Pinned_Arena()->free(recv_buffer);
     }
+#endif
 
     for (int ibeam = 0; ibeam < beams.get_nbeams(); ibeam++) {
         // Loop over particle boxes
         for (BeamParticleIterator pti(beams.getBeam(ibeam), lev); pti.isValid(); ++pti)
         {
-
             // Extract particle properties
             const auto& soa = pti.GetStructOfArrays(); // For momenta and weights
             const auto uzp = soa.GetRealData(BeamIdx::uz).data();
@@ -199,11 +199,16 @@ AdaptiveTimeStep::Calculate (amrex::Real& dt, const int nt, MultiBeam& beams,
         /* For serial runs, set adaptive time step right away */
         if (numprocs_z == 1) dt = new_dt;
 
-        PassTimeStepInfoFinish();
-        m_send_buffer = (amrex::Real*)amrex::The_Pinned_Arena()->alloc(sizeof(amrex::Real));
-        m_send_buffer[WhichDouble::Dt] = new_dt;
-        MPI_Isend(m_send_buffer, 1, amrex::ParallelDescriptor::Mpi_typemap<amrex::Real>::type(),
-                 numprocs_z-1, comm_z_tag, a_comm_z, &m_send_request);
-
+#ifdef AMREX_USE_MPI
+        if (numprocs_z > 1) {
+            PassTimeStepInfoFinish();
+            if (nt < Hipace::m_max_step -numprocs_z -1 ) {
+                m_send_buffer = (amrex::Real*)amrex::The_Pinned_Arena()->alloc(sizeof(amrex::Real));
+                m_send_buffer[WhichDouble::Dt] = new_dt;
+                MPI_Isend(m_send_buffer, 1, amrex::ParallelDescriptor::Mpi_typemap<amrex::Real>::type(),
+                     numprocs_z-1, comm_z_tag, a_comm_z, &m_send_request);
+            }
+        }
+#endif
     }
 }