diff --git a/src/Hipace.H b/src/Hipace.H index ac1817cf8f..4783405920 100644 --- a/src/Hipace.H +++ b/src/Hipace.H @@ -159,7 +159,7 @@ public: /** Particle container for the plasma */ PlasmaParticleContainer m_plasma_container; /** Number of time iterations */ - int m_max_step = 0; + static int m_max_step; /** Time step for the beam evolution */ static amrex::Real m_dt; /** Number of iterations between consecutive output dumps. diff --git a/src/Hipace.cpp b/src/Hipace.cpp index 688a001353..3dd3301e6a 100644 --- a/src/Hipace.cpp +++ b/src/Hipace.cpp @@ -23,6 +23,7 @@ namespace { Hipace* Hipace::m_instance = nullptr; +int Hipace::m_max_step = 0; amrex::Real Hipace::m_dt = 0.0; bool Hipace::m_normalized_units = false; int Hipace::m_verbose = 0; diff --git a/src/utils/AdaptiveTimeStep.cpp b/src/utils/AdaptiveTimeStep.cpp index ef9fb56ad3..70a370a6cd 100644 --- a/src/utils/AdaptiveTimeStep.cpp +++ b/src/utils/AdaptiveTimeStep.cpp @@ -25,11 +25,12 @@ void AdaptiveTimeStep::PassTimeStepInfo (const int nt, MPI_Comm a_comm_z) { HIPACE_PROFILE("SetTimeStep()"); - +#ifdef AMREX_USE_MPI if (m_do_adaptive_time_step == 0) return; const int my_rank_z = amrex::ParallelDescriptor::MyProc(); const int numprocs_z = amrex::ParallelDescriptor::NProcs(); + if (numprocs_z == 0) return; if ((nt % numprocs_z == 0) && (my_rank_z >= 1)) { @@ -45,6 +46,7 @@ AdaptiveTimeStep::PassTimeStepInfo (const int nt, MPI_Comm a_comm_z) my_rank_z-1, comm_z_tag, a_comm_z, &m_send_request); } +#endif } void @@ -78,13 +80,13 @@ AdaptiveTimeStep::Calculate (amrex::Real& dt, const int nt, MultiBeam& beams, // Extract properties associated with physical size of the box const PhysConst phys_const = get_phys_const(); - if (my_rank_z == numprocs_z-1) { - m_timestep_data[WhichDouble::SumWeights] = 0.; - m_timestep_data[WhichDouble::SumWeightsTimesUz] = 0.; - m_timestep_data[WhichDouble::SumWeightsTimesUzSquared] = 0.; - m_timestep_data[WhichDouble::MinUz] = 1e30; - if (my_rank_z == numprocs_z -1 ) - { +#ifdef AMREX_USE_MPI + if (numprocs_z > 1) { + if (my_rank_z == numprocs_z-1) { + m_timestep_data[WhichDouble::SumWeights] = 0.; + m_timestep_data[WhichDouble::SumWeightsTimesUz] = 0.; + m_timestep_data[WhichDouble::SumWeightsTimesUzSquared] = 0.; + m_timestep_data[WhichDouble::MinUz] = 1e30; if (nt > 0){ // first rank receives the new dt from last rank auto recv_buffer = (amrex::Real*)amrex::The_Pinned_Arena()->alloc( @@ -98,31 +100,29 @@ AdaptiveTimeStep::Calculate (amrex::Real& dt, const int nt, MultiBeam& beams, } /* always set time step, because it will be passed */ m_timestep_data[WhichDouble::Dt] = dt; - - } - - } else { - /* lower ranks receive all time step data from the upper rank */ - auto recv_buffer = (amrex::Real*)amrex::The_Pinned_Arena()->alloc - (sizeof(amrex::Real)*WhichDouble::N); - MPI_Status status; - MPI_Recv(recv_buffer, WhichDouble::N, - amrex::ParallelDescriptor::Mpi_typemap::type(), - my_rank_z+1, comm_z_tag, a_comm_z, &status); - - for (int idouble=0; idouble<(int) WhichDouble::N; idouble++) { - m_timestep_data[idouble] = recv_buffer[idouble]; + } else { + /* lower ranks receive all time step data from the upper rank */ + auto recv_buffer = (amrex::Real*)amrex::The_Pinned_Arena()->alloc + (sizeof(amrex::Real)*WhichDouble::N); + MPI_Status status; + MPI_Recv(recv_buffer, WhichDouble::N, + amrex::ParallelDescriptor::Mpi_typemap::type(), + my_rank_z+1, comm_z_tag, a_comm_z, &status); + + for (int idouble=0; idouble<(int) WhichDouble::N; idouble++) { + m_timestep_data[idouble] = recv_buffer[idouble]; + } + /* setting dt, so it can be used in the beam pusher */ + dt = recv_buffer[WhichDouble::Dt]; + amrex::The_Pinned_Arena()->free(recv_buffer); } - /* setting dt, so it can be used in the beam pusher */ - dt = recv_buffer[WhichDouble::Dt]; - amrex::The_Pinned_Arena()->free(recv_buffer); } +#endif for (int ibeam = 0; ibeam < beams.get_nbeams(); ibeam++) { // Loop over particle boxes for (BeamParticleIterator pti(beams.getBeam(ibeam), lev); pti.isValid(); ++pti) { - // Extract particle properties const auto& soa = pti.GetStructOfArrays(); // For momenta and weights const auto uzp = soa.GetRealData(BeamIdx::uz).data(); @@ -199,11 +199,16 @@ AdaptiveTimeStep::Calculate (amrex::Real& dt, const int nt, MultiBeam& beams, /* For serial runs, set adaptive time step right away */ if (numprocs_z == 1) dt = new_dt; - PassTimeStepInfoFinish(); - m_send_buffer = (amrex::Real*)amrex::The_Pinned_Arena()->alloc(sizeof(amrex::Real)); - m_send_buffer[WhichDouble::Dt] = new_dt; - MPI_Isend(m_send_buffer, 1, amrex::ParallelDescriptor::Mpi_typemap::type(), - numprocs_z-1, comm_z_tag, a_comm_z, &m_send_request); - +#ifdef AMREX_USE_MPI + if (numprocs_z > 1) { + PassTimeStepInfoFinish(); + if (nt < Hipace::m_max_step -numprocs_z -1 ) { + m_send_buffer = (amrex::Real*)amrex::The_Pinned_Arena()->alloc(sizeof(amrex::Real)); + m_send_buffer[WhichDouble::Dt] = new_dt; + MPI_Isend(m_send_buffer, 1, amrex::ParallelDescriptor::Mpi_typemap::type(), + numprocs_z-1, comm_z_tag, a_comm_z, &m_send_request); + } + } +#endif } }