From 34ecd8ba6c8737d48ce423ec73c043a3345ada1c Mon Sep 17 00:00:00 2001 From: AlexanderSinn Date: Wed, 27 Mar 2024 19:27:30 +0100 Subject: [PATCH 1/2] Use 64 bits for Beam ID --- docs/source/run/parameters.rst | 3 -- src/diagnostics/OpenPMDWriter.cpp | 4 +- src/particles/beam/BeamParticleContainer.H | 3 +- src/particles/beam/BeamParticleContainer.cpp | 10 ++--- .../beam/BeamParticleContainerInit.cpp | 37 ++++++++++--------- .../deposition/BeamDepositCurrent.cpp | 6 ++- .../plasma/PlasmaParticleContainer.cpp | 5 +-- .../plasma/PlasmaParticleContainerInit.cpp | 10 +---- 8 files changed, 35 insertions(+), 43 deletions(-) diff --git a/docs/source/run/parameters.rst b/docs/source/run/parameters.rst index 8f188c91c0..b0f249f77a 100644 --- a/docs/source/run/parameters.rst +++ b/docs/source/run/parameters.rst @@ -514,9 +514,6 @@ which are valid only for certain beam types, are introduced further below under Whether the beam particles are pushed along the z-axis. The momentum is still fully updated. Note: using ``do_z_push = 0`` results in unphysical behavior. -* `` or beams.do_reset_id_init`` (`bool`) optional (default `0`) - Whether to reset the ID incrementor to 1 before initializing beam particles. - * `` or beams.reorder_period`` (`int`) optional (default `0`) Reorder particles periodically to speed-up current deposition and particle push on GPU. A good starting point is a period of 1 to reorder beam particles on every timestep. diff --git a/src/diagnostics/OpenPMDWriter.cpp b/src/diagnostics/OpenPMDWriter.cpp index 17ba17cf3f..4f91f039d8 100644 --- a/src/diagnostics/OpenPMDWriter.cpp +++ b/src/diagnostics/OpenPMDWriter.cpp @@ -249,7 +249,9 @@ OpenPMDWriter::WriteBeamParticleData (MultiBeam& beams, openPMD::Iteration itera uint64_t * const uint64_data = m_uint64_beam_data[ibeam][idx].get(); for (uint64_t i=0; i= 1, "n_subcycles must be >= 1"); - queryWithParserAlt(pp, "do_reset_id_init", m_do_reset_id_init, pp_alt); queryWithParser(pp, "do_salame", m_do_salame); queryWithParserAlt(pp, "reorder_period", m_reorder_period, pp_alt); amrex::Array idx_array @@ -121,7 +120,6 @@ BeamParticleContainer::InitData (const amrex::Geometry& geom) amrex::ParmParse pp(m_name); amrex::ParmParse pp_alt("beams"); amrex::Real ptime {0.}; - if (m_do_reset_id_init) BeamTileInit::ParticleType::NextID(1); if (m_injection_type == "fixed_ppc") { queryWithParser(pp, "ppc", m_ppc); @@ -315,7 +313,7 @@ void BeamParticleContainer::TagByLevel (const int current_N_level, auto& soa = getBeamSlice(which_slice).GetStructOfArrays(); const amrex::Real * const pos_x = soa.GetRealData(BeamIdx::x).data(); const amrex::Real * const pos_y = soa.GetRealData(BeamIdx::y).data(); - auto * const idcpup = soa.GetIdCPUData().data(); + int * const p_mr_level = soa.GetIntData(BeamIdx::mr_level).data(); const int lev1_idx = std::min(1, current_N_level-1); const int lev2_idx = std::min(2, current_N_level-1); @@ -341,15 +339,15 @@ void BeamParticleContainer::TagByLevel (const int current_N_level, lo_x_lev2 < xp && xp < hi_x_lev2 && lo_y_lev2 < yp && yp < hi_y_lev2) { // level 2 - amrex::ParticleCPUWrapper{idcpup[ip]} = 2; + p_mr_level[ip] = 2; } else if (current_N_level > 1 && lo_x_lev1 < xp && xp < hi_x_lev1 && lo_y_lev1 < yp && yp < hi_y_lev1) { // level 1 - amrex::ParticleCPUWrapper{idcpup[ip]} = 1; + p_mr_level[ip] = 1; } else { // level 0 - amrex::ParticleCPUWrapper{idcpup[ip]} = 0; + p_mr_level[ip] = 0; } } ); diff --git a/src/particles/beam/BeamParticleContainerInit.cpp b/src/particles/beam/BeamParticleContainerInit.cpp index 4b043a3d89..6935caee71 100644 --- a/src/particles/beam/BeamParticleContainerInit.cpp +++ b/src/particles/beam/BeamParticleContainerInit.cpp @@ -41,8 +41,8 @@ namespace void AddOneBeamParticle ( const BeamTileInit::ParticleTileDataType& ptd, const amrex::Real& x, const amrex::Real& y, const amrex::Real& z, const amrex::Real& ux, const amrex::Real& uy, - const amrex::Real& uz, const amrex::Real& weight, const int& pid, - const int& ip, const amrex::Real& speed_of_light) noexcept + const amrex::Real& uz, const amrex::Real& weight, const amrex::Long pid, + const amrex::Long ip, const amrex::Real& speed_of_light) noexcept { ptd.rdata(BeamIdx::x )[ip] = x; ptd.rdata(BeamIdx::y )[ip] = y; @@ -52,8 +52,8 @@ namespace ptd.rdata(BeamIdx::uz )[ip] = uz * speed_of_light; ptd.rdata(BeamIdx::w )[ip] = std::abs(weight); - ptd.id(ip) = pid > 0 ? pid + ip : pid; - ptd.cpu(ip) = 0; + ptd.idcpu(ip) = pid + ip; + ptd.id(ip).make_valid(); } /** \brief Adds a single beam particle into the per-slice BeamTile @@ -86,15 +86,15 @@ namespace ptd.rdata(BeamIdx::uz )[ip] = uz * speed_of_light; ptd.rdata(BeamIdx::w )[ip] = std::abs(weight); - // ensure id is always valid - // it will repeat if there are more than 2^31-1 particles - int id = int((pid + ip) & ((1ull << 31) - 1)); - if (id == 0) id = 1; - if (!is_valid) id = -1; + ptd.idcpu(ip) = pid + ip; + if (is_valid) { + ptd.id(ip).make_valid(); // ensure id is valid + } else { + ptd.id(ip).make_invalid(); + } - ptd.id(ip) = id; - ptd.cpu(ip) = 0; ptd.idata(BeamIdx::nsubcycles)[ip] = 0; + ptd.idata(BeamIdx::mr_level)[ip] = 0; } } @@ -389,7 +389,7 @@ InitBeamFixedWeightSlice (int slice, int which_slice) amrex::Real * const pos_z = m_z_array.dataPtr(); const uint64_t pid = m_id64; - m_id64 += num_to_add; + m_id64 += m_do_symmetrize ? 4*num_to_add : num_to_add; const bool can = m_can_profile; const bool do_symmetrize = m_do_symmetrize; @@ -587,7 +587,7 @@ InitBeamFixedWeightPDFSlice (int slice, int which_slice) const auto ptd = particle_tile.getParticleTileData(); const uint64_t pid = m_id64; - m_id64 += num_to_add; + m_id64 += m_do_symmetrize ? 4*num_to_add : num_to_add; const amrex::Real clight = get_phys_const().c; const bool do_symmetrize = m_do_symmetrize; @@ -947,7 +947,7 @@ InitBeamFromFile (const std::string input_file, const auto num_to_add = electrons[name_r][name_rx].getExtent()[0]; if(num_to_add >= 2147483647) { - amrex::Abort("Beam can't have more than 2'147'483'646 Particles\n"); + amrex::Abort("Beam from file can't have more than 2'147'483'646 Particles\n"); } auto del = [](input_type *p){ amrex::The_Pinned_Arena()->free(reinterpret_cast(p)); }; @@ -1031,8 +1031,9 @@ InitBeamFromFile (const std::string input_file, auto new_size = old_size + num_to_add; particle_tile.resize(new_size); const auto ptd = particle_tile.getParticleTileData(); - const int pid = BeamTileInit::ParticleType::NextID(); - BeamTileInit::ParticleType::NextID(pid + num_to_add); + + const uint64_t pid = m_id64; + m_id64 += num_to_add; const input_type * const r_x_ptr = r_x_data.get(); const input_type * const r_y_ptr = r_y_data.get(); @@ -1042,8 +1043,8 @@ InitBeamFromFile (const std::string input_file, const input_type * const u_z_ptr = u_z_data.get(); const input_type * const w_w_ptr = w_w_data.get(); - amrex::ParallelFor(int(num_to_add), - [=] AMREX_GPU_DEVICE (const int i) { + amrex::ParallelFor(amrex::Long(num_to_add), + [=] AMREX_GPU_DEVICE (const amrex::Long i) { AddOneBeamParticle(ptd, static_cast(r_x_ptr[i] * unit_rx), static_cast(r_y_ptr[i] * unit_ry), diff --git a/src/particles/deposition/BeamDepositCurrent.cpp b/src/particles/deposition/BeamDepositCurrent.cpp index 494d2803ae..eaa81891e7 100644 --- a/src/particles/deposition/BeamDepositCurrent.cpp +++ b/src/particles/deposition/BeamDepositCurrent.cpp @@ -99,7 +99,11 @@ DepositCurrentSlice (BeamParticleContainer& beam, Fields& fields, // Skip invalid particles and ghost particles not in the last slice // beam deposits only up to its finest level - if (!ptd.id(ip).is_valid() || (only_highest ? (ptd.cpu(ip)!=lev) : (ptd.cpu(ip) Date: Wed, 27 Mar 2024 19:50:22 +0100 Subject: [PATCH 2/2] =?UTF-8?q?Change=20id=20in=20CI=20test=20because=20id?= =?UTF-8?q?s=20with=C2=A0do=5Fsymmetrize=20dont=20repeat=20anymore?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/particles/beam/BeamParticleContainer.H | 2 +- tests/checksum/benchmarks_json/production.SI.2Rank_pwfa.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/particles/beam/BeamParticleContainer.H b/src/particles/beam/BeamParticleContainer.H index b1a5249d84..c09412769f 100644 --- a/src/particles/beam/BeamParticleContainer.H +++ b/src/particles/beam/BeamParticleContainer.H @@ -203,7 +203,7 @@ public: return rcomp < numRealComponents(); } bool communicateIntComponent (int icomp) const { - // don't communicate nsubcycles + // don't communicate nsubcycles or mr_level return icomp < BeamIdx::int_nattribs_in_buffer; } diff --git a/tests/checksum/benchmarks_json/production.SI.2Rank_pwfa.json b/tests/checksum/benchmarks_json/production.SI.2Rank_pwfa.json index 9c2c191004..6b613e7699 100644 --- a/tests/checksum/benchmarks_json/production.SI.2Rank_pwfa.json +++ b/tests/checksum/benchmarks_json/production.SI.2Rank_pwfa.json @@ -19,7 +19,7 @@ }, "driver": { "charge": 1.601913877032e-13, - "id": 137641610358, + "id": 499836513366, "mass": 9.107889762573e-25, "x": 3.1337620903426, "y": 3.1349173868172, @@ -31,7 +31,7 @@ }, "witness": { "charge": 1.602176634e-13, - "id": 199678708984, + "id": 500000500000, "mass": 9.1093837015e-25, "x": 2.457586712044, "y": 2.4574680947762,