From 9986eefce6279a52b0a7ad99e14c3923547c2bc6 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 23 Sep 2024 12:08:01 -0700 Subject: [PATCH 01/10] Named SoA Support Add support to optionally name SoA Real and Int components, both for compile-time and runtime components. --- Src/Particle/AMReX_ParticleContainer.H | 35 +++++++++++- Src/Particle/AMReX_ParticleContainerI.H | 51 ++++++++++++++++- Src/Particle/AMReX_ParticleTile.H | 12 +++- Src/Particle/AMReX_StructOfArrays.H | 74 ++++++++++++++++++++++++- 4 files changed, 163 insertions(+), 9 deletions(-) diff --git a/Src/Particle/AMReX_ParticleContainer.H b/Src/Particle/AMReX_ParticleContainer.H index 03a2254a10e..e59ccb89da2 100644 --- a/Src/Particle/AMReX_ParticleContainer.H +++ b/Src/Particle/AMReX_ParticleContainer.H @@ -60,6 +60,7 @@ #include #include #include +#include #include #include #include @@ -1144,7 +1145,8 @@ public: */ ParticleTileType& DefineAndReturnParticleTile (int lev, int grid, int tile) { - m_particles[lev][std::make_pair(grid, tile)].define(NumRuntimeRealComps(), NumRuntimeIntComps()); + m_particles[lev][std::make_pair(grid, tile)].define(NumRuntimeRealComps(), NumRuntimeIntComps(), &m_soa_rdata_names, &m_soa_idata_names); + return ParticlesAt(lev, grid, tile); } @@ -1249,8 +1251,10 @@ public: template ,int> = 0> - void AddRealComp (T communicate=true) + void AddRealComp (std::string const & name, T communicate=true) { + m_soa_rdata_names.push_back(name); + m_runtime_comps_defined = true; m_num_runtime_real++; h_redistribute_real_comp.push_back(communicate); @@ -1272,8 +1276,18 @@ public: template ,int> = 0> - void AddIntComp (T communicate=true) + void AddRealComp (T communicate=true) + { + std::string default_name = "make-up-name"; //! @FIXME add naming scheme + AddRealComp(default_name, communicate); + } + + template ,int> = 0> + void AddIntComp (std::string const & name, T communicate=true) { + m_soa_idata_names.push_back(name); + m_runtime_comps_defined = true; m_num_runtime_int++; h_redistribute_int_comp.push_back(communicate); @@ -1293,6 +1307,14 @@ public: } } + template ,int> = 0> + void AddIntComp (T communicate=true) + { + std::string default_name = "make-up-name"; //! @FIXME add naming scheme + AddIntComp(default_name, communicate); + } + int NumRuntimeRealComps () const { return m_num_runtime_real; } int NumRuntimeIntComps () const { return m_num_runtime_int; } @@ -1428,6 +1450,9 @@ private: void Initialize (); + /** Overwrite the default names for the compile-time SoA components */ + void SetSoACompileTimeNames (std::vector const & rdata_name, std::vector const & idata_name); + bool m_runtime_comps_defined{false}; int m_num_runtime_real{0}; int m_num_runtime_int{0}; @@ -1435,6 +1460,10 @@ private: size_t particle_size, superparticle_size; int num_real_comm_comps, num_int_comm_comps; Vector m_particles; + + // names of both compile-time and runtime Real and Int SoA data + std::vector m_soa_rdata_names; + std::vector m_soa_idata_names; }; template class Allocator, class CellAssignor> diff --git a/Src/Particle/AMReX_ParticleContainerI.H b/Src/Particle/AMReX_ParticleContainerI.H index 74e65b792f0..6cd7cb6159c 100644 --- a/Src/Particle/AMReX_ParticleContainerI.H +++ b/Src/Particle/AMReX_ParticleContainerI.H @@ -1,6 +1,10 @@ -#include #include +#include +#include +#include + + template class Allocator, class CellAssignor> void @@ -60,10 +64,51 @@ ParticleContainer_impl class Allocator, class CellAssignor> +void +ParticleContainer_impl :: SetSoACompileTimeNames ( + std::vector const & rdata_name, std::vector const & idata_name +) +{ + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(rdata_name.size() == NArrayReal, "rdata_name must be equal to NArrayReal"); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(idata_name.size() == NArrayInt, "idata_name must be equal to NArrayInt"); + + for (int i=0; i class Allocator, class CellAssignor> template @@ -1161,7 +1206,7 @@ ParticleContainer_impl #include +#include #include +#include + namespace amrex { @@ -730,10 +733,15 @@ struct ParticleTile ParticleTile& operator= (ParticleTile &&) noexcept = default; #endif - void define (int a_num_runtime_real, int a_num_runtime_int) + void define ( + int a_num_runtime_real, + int a_num_runtime_int, + std::vector* soa_rdata_names=nullptr, + std::vector* soa_idata_names=nullptr + ) { m_defined = true; - GetStructOfArrays().define(a_num_runtime_real, a_num_runtime_int); + GetStructOfArrays().define(a_num_runtime_real, a_num_runtime_int, soa_rdata_names, soa_idata_names); m_runtime_r_ptrs.resize(a_num_runtime_real); m_runtime_i_ptrs.resize(a_num_runtime_int); m_runtime_r_cptrs.resize(a_num_runtime_real); diff --git a/Src/Particle/AMReX_StructOfArrays.H b/Src/Particle/AMReX_StructOfArrays.H index 4de35e085ca..201cf340dc1 100644 --- a/Src/Particle/AMReX_StructOfArrays.H +++ b/Src/Particle/AMReX_StructOfArrays.H @@ -6,7 +6,11 @@ #include #include +#include #include +#include +#include + namespace amrex { @@ -19,11 +23,18 @@ struct StructOfArrays { using RealVector = amrex::PODVector >; using IntVector = amrex::PODVector >; - void define (int a_num_runtime_real, int a_num_runtime_int) + void define ( + int a_num_runtime_real, + int a_num_runtime_int, + std::vector* soa_rdata_names=nullptr, + std::vector* soa_idata_names=nullptr + ) { m_defined = true; m_runtime_rdata.resize(a_num_runtime_real); m_runtime_idata.resize(a_num_runtime_int ); + m_rdata_names = soa_rdata_names; + m_idata_names = soa_idata_names; } [[nodiscard]] int NumRealComps () const noexcept { return NReal + m_runtime_rdata.size(); } @@ -79,6 +90,32 @@ struct StructOfArrays { } } + /** Get access to a particle Real component Array (compile-time and runtime component) + * + * @param name named component component with 0...NReal-1 compile-time and NReal... runtime arguments + */ + [[nodiscard]] RealVector& GetRealData (std::string const & name) { + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(m_rdata_names != nullptr, "SoA Real names were not defined."); + auto const pos = std::find(m_rdata_names->begin(), m_rdata_names->end(), name); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(pos != m_rdata_names->end(), "Soa Real name='" + name + "' was not found components"); + + int const index = std::distance(m_rdata_names->begin(), pos); + return GetRealData(index); + } + + /** Get access to a particle Real component Array (compile-time and runtime component) + * + * @param name named component component with 0...NReal-1 compile-time and NReal... runtime arguments + */ + [[nodiscard]] const RealVector& GetRealData (std::string const & name) const { + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(m_rdata_names != nullptr, "SoA Real names were not defined."); + auto const pos = std::find(m_rdata_names->begin(), m_rdata_names->end(), name); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(pos != m_rdata_names->end(), "Soa Real name='" + name + "' was not found components"); + + int const index = std::distance(m_rdata_names->begin(), pos); + return GetRealData(index); + } + /** Get access to a particle Int component Array (compile-time and runtime component) * * @param index component with 0...NInt-1 compile-time and NInt... runtime arguments @@ -118,6 +155,34 @@ struct StructOfArrays { } } + /** Get access to a particle Int component Array (compile-time and runtime component) + * + * @param index component with 0...NInt-1 compile-time and NInt... runtime arguments + * @return + */ + [[nodiscard]] IntVector& GetIntData (std::string const & name) { + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(m_idata_names != nullptr, "SoA Int names were not defined."); + auto const pos = std::find(m_idata_names->begin(), m_idata_names->end(), name); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(pos != m_idata_names->end(), "Soa Int name='" + name + "' was not found components"); + + int const index = std::distance(m_idata_names->begin(), pos); + return GetIntData(index); + } + + /** Get access to a particle Int component Array (compile-time and runtime component) + * + * @param index component with 0...NInt-1 compile-time and NInt... runtime arguments + * @return + */ + [[nodiscard]] const IntVector& GetIntData (std::string const & name) const { + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(m_idata_names != nullptr, "SoA Int names were not defined."); + auto const pos = std::find(m_idata_names->begin(), m_idata_names->end(), name); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(pos != m_idata_names->end(), "Soa Int name='" + name + "' was not found components"); + + int const index = std::distance(m_idata_names->begin(), pos); + return GetIntData(index); + } + /** * \brief Returns the total number of particles (real and neighbor) * @@ -226,13 +291,20 @@ struct StructOfArrays { int m_num_neighbor_particles{0}; private: + // compile-time data IdCPU m_idcpu; std::array m_rdata; std::array< IntVector, NInt> m_idata; + // runtime data std::vector m_runtime_rdata; std::vector m_runtime_idata; + // names of both compile-time and runtime Real and Int data + std::vector* m_rdata_names = nullptr; + std::vector* m_idata_names = nullptr; + + //! whether the runtime components are sized correctly bool m_defined{false}; }; From 91785b85c620384be12b18292e685b1d3a2415a7 Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Tue, 24 Sep 2024 12:13:15 -0700 Subject: [PATCH 02/10] more work on named SOA components --- Src/Particle/AMReX_ParticleContainer.H | 32 ++++++++++++-------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/Src/Particle/AMReX_ParticleContainer.H b/Src/Particle/AMReX_ParticleContainer.H index e59ccb89da2..646c52a1411 100644 --- a/Src/Particle/AMReX_ParticleContainer.H +++ b/Src/Particle/AMReX_ParticleContainer.H @@ -1249,9 +1249,7 @@ public: Long superParticleSize() const { return superparticle_size; } - template ,int> = 0> - void AddRealComp (std::string const & name, T communicate=true) + void AddRealComp (std::string const & name, bool communicate=true) { m_soa_rdata_names.push_back(name); @@ -1274,17 +1272,13 @@ public: } } - template ,int> = 0> - void AddRealComp (T communicate=true) + void AddRealComp (bool communicate=true) { - std::string default_name = "make-up-name"; //! @FIXME add naming scheme + std::string default_name = "runtime_real" + std::to_string(m_num_runtime_real); AddRealComp(default_name, communicate); } - template ,int> = 0> - void AddIntComp (std::string const & name, T communicate=true) + void AddIntComp (std::string const & name, bool communicate=true) { m_soa_idata_names.push_back(name); @@ -1307,11 +1301,9 @@ public: } } - template ,int> = 0> - void AddIntComp (T communicate=true) + void AddIntComp (bool communicate=true) { - std::string default_name = "make-up-name"; //! @FIXME add naming scheme + std::string default_name = "runtime_int" + std::to_string(m_num_runtime_int); AddIntComp(default_name, communicate); } @@ -1425,6 +1417,15 @@ public: #include "AMReX_ParticlesHDF5.H" #endif + /** Overwrite the default names for the compile-time SoA components */ + void SetSoACompileTimeNames (std::vector const & rdata_name, std::vector const & idata_name); + + /** Get the names for the real SoA components **/ + std::vector GetRealSoANames () const {return m_soa_rdata_names;} + + /** Get the names for the int SoA components **/ + std::vector GetIntSoANames () const {return m_soa_idata_names;} + protected: template @@ -1450,9 +1451,6 @@ private: void Initialize (); - /** Overwrite the default names for the compile-time SoA components */ - void SetSoACompileTimeNames (std::vector const & rdata_name, std::vector const & idata_name); - bool m_runtime_comps_defined{false}; int m_num_runtime_real{0}; int m_num_runtime_int{0}; From 8a7fd219395e9dde08dfd18e783a5e575ec495e2 Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Tue, 24 Sep 2024 12:13:25 -0700 Subject: [PATCH 03/10] create test --- .../NamedSoAComponents/CMakeLists.txt | 10 + .../Particles/NamedSoAComponents/GNUmakefile | 22 ++ .../Particles/NamedSoAComponents/Make.package | 1 + Tests/Particles/NamedSoAComponents/main.cpp | 216 ++++++++++++++++++ 4 files changed, 249 insertions(+) create mode 100644 Tests/Particles/NamedSoAComponents/CMakeLists.txt create mode 100644 Tests/Particles/NamedSoAComponents/GNUmakefile create mode 100644 Tests/Particles/NamedSoAComponents/Make.package create mode 100644 Tests/Particles/NamedSoAComponents/main.cpp diff --git a/Tests/Particles/NamedSoAComponents/CMakeLists.txt b/Tests/Particles/NamedSoAComponents/CMakeLists.txt new file mode 100644 index 00000000000..e14ddd68975 --- /dev/null +++ b/Tests/Particles/NamedSoAComponents/CMakeLists.txt @@ -0,0 +1,10 @@ +foreach(D IN LISTS AMReX_SPACEDIM) + set(_sources main.cpp) + #set(_input_files) + #set(_input_files inputs) + + setup_test(${D} _sources _input_files NTHREADS 2) + + unset(_sources) + unset(_input_files) +endforeach() diff --git a/Tests/Particles/NamedSoAComponents/GNUmakefile b/Tests/Particles/NamedSoAComponents/GNUmakefile new file mode 100644 index 00000000000..9f49d3ec029 --- /dev/null +++ b/Tests/Particles/NamedSoAComponents/GNUmakefile @@ -0,0 +1,22 @@ +AMREX_HOME = ../../../ + +DEBUG = FALSE + +DIM = 3 + +COMP = gcc + +USE_MPI = TRUE +USE_OMP = FALSE +USE_CUDA = FALSE + +#TINY_PROFILE = TRUE +USE_PARTICLES = TRUE + +include $(AMREX_HOME)/Tools/GNUMake/Make.defs + +include ./Make.package +include $(AMREX_HOME)/Src/Base/Make.package +include $(AMREX_HOME)/Src/Particle/Make.package + +include $(AMREX_HOME)/Tools/GNUMake/Make.rules diff --git a/Tests/Particles/NamedSoAComponents/Make.package b/Tests/Particles/NamedSoAComponents/Make.package new file mode 100644 index 00000000000..6b4b865e8fc --- /dev/null +++ b/Tests/Particles/NamedSoAComponents/Make.package @@ -0,0 +1 @@ +CEXE_sources += main.cpp diff --git a/Tests/Particles/NamedSoAComponents/main.cpp b/Tests/Particles/NamedSoAComponents/main.cpp new file mode 100644 index 00000000000..b3a856c0c66 --- /dev/null +++ b/Tests/Particles/NamedSoAComponents/main.cpp @@ -0,0 +1,216 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +using namespace amrex; + +template class Allocator=DefaultAllocator> +void addParticles () +{ + int is_per[AMREX_SPACEDIM]; + for (int & d : is_per) { + d = 1; + } + + RealBox real_box; + for (int n = 0; n < AMREX_SPACEDIM; n++) + { + real_box.setLo(n, 0.0); + real_box.setHi(n, 100.0); + } + + IntVect domain_lo(AMREX_D_DECL(0, 0, 0)); + IntVect domain_hi(AMREX_D_DECL(127, 127, 127)); + const Box base_domain(domain_lo, domain_hi); + + Geometry geom(base_domain, &real_box, CoordSys::cartesian, is_per); + BoxArray ba(base_domain); + ba.maxSize(64); + + DistributionMapping dm(ba); + + T_PC pc(geom, dm, ba); + + amrex::Print() << "Original Real SoA component names are: "; + for (auto& n : pc.GetRealSoANames()) { + amrex::Print() << n << ", "; + } + amrex::Print() << "\n"; + + amrex::Print() << "Original Int SoA component names are: "; + for (auto& n : pc.GetIntSoANames()) { + amrex::Print() << n << ", "; + } + amrex::Print() << "\n"; + + amrex::Print() << "Adding runtime comps. \n"; + pc.AddRealComp("runtime_rcomp0"); + pc.AddRealComp(); // without name - should be runtime_rcomp1 + pc.AddIntComp(); // without name - should be runtime_icomp0 + + amrex::Print() << "New Real SoA component names are: "; + for (auto& n : pc.GetRealSoANames()) { + amrex::Print() << n << ", "; + } + amrex::Print() << "\n"; + + amrex::Print() << "New Int SoA component names are: "; + for (auto& n : pc.GetIntSoANames()) { + amrex::Print() << n << ", "; + } + amrex::Print() << "\n"; + + amrex::Print() << "Reset compile-time SoA names \n"; + pc.SetSoACompileTimeNames({"x", "y", "z", "w"}, {"i1", "i2"}); + + amrex::Print() << "New Real SoA component names are: "; + for (auto& n : pc.GetRealSoANames()) { + amrex::Print() << n << ", "; + } + amrex::Print() << "\n"; + + amrex::Print() << "New Int SoA component names are: "; + for (auto& n : pc.GetIntSoANames()) { + amrex::Print() << n << ", "; + } + amrex::Print() << "\n"; + + int const NArrayReal = pc.NArrayReal; + int const NArrayInt = pc.NArrayInt; + + using ParticleType = typename T_PC::ParticleType; + using ParticleTileDataType = typename T_PC::ParticleTileType::ParticleTileDataType; + + const int add_num_particles = 5; + + auto& ptile1 = pc.DefineAndReturnParticleTile(0, 0, 0); + ptile1.resize(add_num_particles); + + for (int i = 0; i < add_num_particles; ++i) + { + for (int d = 0; d < AMREX_SPACEDIM; d++) { + ptile1.pos(i, d) = 12.0; + } + ptile1.getParticleTileData().rdata(AMREX_SPACEDIM)[i] = 1.2; // w + + ptile1.push_back_int(0, ParticleType::NextID()); + ptile1.push_back_int(1, amrex::ParallelDescriptor::MyProc()); + } + + int lev=0; + using MyParIter = ParIter_impl; + for (MyParIter pti(pc, lev); pti.isValid(); ++pti) { + const int np = pti.numParticles(); + // preparing access to particle data: SoA of Reals + auto& soa = pti.GetStructOfArrays(); + auto xp = soa.GetRealData("x").data(); + auto yp = soa.GetRealData("y").data(); + auto zp = soa.GetRealData("z").data(); + auto wp = soa.GetRealData("w").data(); + auto soa_real = soa.GetRealData(); + auto size = soa.size(); + amrex::ParticleReal* const AMREX_RESTRICT part_x = soa_real[0].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT part_y = AMREX_SPACEDIM >= 2 ? soa_real[1].dataPtr() : nullptr; + amrex::ParticleReal* const AMREX_RESTRICT part_z = AMREX_SPACEDIM >= 3 ? soa_real[2].dataPtr() : nullptr; + amrex::ParticleReal* const AMREX_RESTRICT part_w = soa_real[AMREX_SPACEDIM].dataPtr(); + auto& soa_int = pti.GetStructOfArrays().GetIntData(); + amrex::ignore_unused(size, part_x, part_y, part_z, part_w, soa_int); + + // Iterating over SoA Particles + ParticleTileDataType ptd = pti.GetParticleTile().getParticleTileData(); + + ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) + { + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(xp[ip] == 12_prt, + "pos attribute expected to be 12"); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(yp[ip] == 12_prt, + "pos attribute expected to be 12"); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(zp[ip] == 12_prt, + "pos attribute expected to be 12"); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(wp[ip] == 1.2_prt, + "pos attribute expected to be 1.2"); + }); + + + } + + // create a host-side particle buffer + auto tmp = pc.template make_alike(); + tmp.copyParticles(pc, true); + + using MyPinnedParIter = ParIter_impl; + + for (MyPinnedParIter pti(tmp, lev); pti.isValid(); ++pti) { + auto& particle_attributes = pti.GetStructOfArrays(); + auto& real_comp0 = particle_attributes.GetRealData(0); + auto& int_comp1 = particle_attributes.GetIntData(1); + for (int i = 0; i < pti.numParticles(); ++i) { + real_comp0[i] += 1; + int_comp1[i] += 1; + } + } + + tmp.Redistribute(); + + using ConstPTDType = typename T_PC::ParticleTileType::ConstParticleTileDataType; + amrex::ReduceOps reduce_ops; + auto r = amrex::ParticleReduce< + amrex::ReduceData< + amrex::ParticleReal, amrex::ParticleReal, amrex::ParticleReal, + amrex::ParticleReal, amrex::ParticleReal, amrex::ParticleReal, + amrex::ParticleReal> + >( + pc, + [=] AMREX_GPU_DEVICE(const ConstPTDType& ptd, const int i) noexcept + { + amrex::ParticleReal const x = ptd.rdata(0)[i]; + amrex::ParticleReal const y = AMREX_SPACEDIM >= 2 ? ptd.rdata(1)[i] : 0.0; + amrex::ParticleReal const z = AMREX_SPACEDIM >= 3 ? ptd.rdata(2)[i] : 0.0; + + amrex::ParticleReal const w = ptd.rdata(AMREX_SPACEDIM)[i]; + + return amrex::makeTuple(x, x*x, y, y*y, z, z*z, w); + }, + reduce_ops + ); + amrex::ignore_unused(r); + + // Reduce for SoA Particle Struct + /* + using PTDType = typename T_PC::ParticleTileType::ConstParticleTileDataType; + amrex::ReduceOps reduce_ops; + auto r = amrex::ParticleReduce> ( + pc, [=] AMREX_GPU_DEVICE (const PTDType& ptd, const int i) noexcept + -> amrex::GpuTuple + { + + const amrex::Real a = ptd.rdata(1)[i]; + const amrex::Real b = ptd.rdata(2)[i]; + const int c = ptd.idata(1)[i]; + return {a, b, c}; + }, reduce_ops); + + AMREX_ALWAYS_ASSERT(amrex::get<0>(r) == amrex::Real(std::pow(256, AMREX_SPACEDIM))); + AMREX_ALWAYS_ASSERT(amrex::get<1>(r) == 2.0); + AMREX_ALWAYS_ASSERT(amrex::get<2>(r) == 1); + */ +} + + + + +int main(int argc, char* argv[]) + { + amrex::Initialize(argc,argv); + { + addParticles< ParticleContainerPureSoA<4, 2> > (); + } + amrex::Finalize(); + } From d6d40cdc89bbe1b3fb059726917e24ea5e536f5e Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Wed, 25 Sep 2024 12:16:04 -0700 Subject: [PATCH 04/10] clean up test --- Tests/Particles/NamedSoAComponents/main.cpp | 92 ++------------------- 1 file changed, 7 insertions(+), 85 deletions(-) diff --git a/Tests/Particles/NamedSoAComponents/main.cpp b/Tests/Particles/NamedSoAComponents/main.cpp index b3a856c0c66..bf9fbad0749 100644 --- a/Tests/Particles/NamedSoAComponents/main.cpp +++ b/Tests/Particles/NamedSoAComponents/main.cpp @@ -11,9 +11,9 @@ using namespace amrex; -template class Allocator=DefaultAllocator> void addParticles () { + using PC = ParticleContainerPureSoA<4, 2>; int is_per[AMREX_SPACEDIM]; for (int & d : is_per) { d = 1; @@ -36,7 +36,7 @@ void addParticles () DistributionMapping dm(ba); - T_PC pc(geom, dm, ba); + PC pc(geom, dm, ba); amrex::Print() << "Original Real SoA component names are: "; for (auto& n : pc.GetRealSoANames()) { @@ -85,8 +85,8 @@ void addParticles () int const NArrayReal = pc.NArrayReal; int const NArrayInt = pc.NArrayInt; - using ParticleType = typename T_PC::ParticleType; - using ParticleTileDataType = typename T_PC::ParticleTileType::ParticleTileDataType; + using ParticleType = typename PC::ParticleType; + using ParticleTileDataType = typename PC::ParticleTileType::ParticleTileDataType; const int add_num_particles = 5; @@ -107,25 +107,13 @@ void addParticles () int lev=0; using MyParIter = ParIter_impl; for (MyParIter pti(pc, lev); pti.isValid(); ++pti) { - const int np = pti.numParticles(); - // preparing access to particle data: SoA of Reals auto& soa = pti.GetStructOfArrays(); auto xp = soa.GetRealData("x").data(); auto yp = soa.GetRealData("y").data(); auto zp = soa.GetRealData("z").data(); auto wp = soa.GetRealData("w").data(); - auto soa_real = soa.GetRealData(); - auto size = soa.size(); - amrex::ParticleReal* const AMREX_RESTRICT part_x = soa_real[0].dataPtr(); - amrex::ParticleReal* const AMREX_RESTRICT part_y = AMREX_SPACEDIM >= 2 ? soa_real[1].dataPtr() : nullptr; - amrex::ParticleReal* const AMREX_RESTRICT part_z = AMREX_SPACEDIM >= 3 ? soa_real[2].dataPtr() : nullptr; - amrex::ParticleReal* const AMREX_RESTRICT part_w = soa_real[AMREX_SPACEDIM].dataPtr(); - auto& soa_int = pti.GetStructOfArrays().GetIntData(); - amrex::ignore_unused(size, part_x, part_y, part_z, part_w, soa_int); - - // Iterating over SoA Particles - ParticleTileDataType ptd = pti.GetParticleTile().getParticleTileData(); + const int np = pti.numParticles(); ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) { AMREX_ALWAYS_ASSERT_WITH_MESSAGE(xp[ip] == 12_prt, @@ -137,80 +125,14 @@ void addParticles () AMREX_ALWAYS_ASSERT_WITH_MESSAGE(wp[ip] == 1.2_prt, "pos attribute expected to be 1.2"); }); - - - } - - // create a host-side particle buffer - auto tmp = pc.template make_alike(); - tmp.copyParticles(pc, true); - - using MyPinnedParIter = ParIter_impl; - - for (MyPinnedParIter pti(tmp, lev); pti.isValid(); ++pti) { - auto& particle_attributes = pti.GetStructOfArrays(); - auto& real_comp0 = particle_attributes.GetRealData(0); - auto& int_comp1 = particle_attributes.GetIntData(1); - for (int i = 0; i < pti.numParticles(); ++i) { - real_comp0[i] += 1; - int_comp1[i] += 1; - } } - - tmp.Redistribute(); - - using ConstPTDType = typename T_PC::ParticleTileType::ConstParticleTileDataType; - amrex::ReduceOps reduce_ops; - auto r = amrex::ParticleReduce< - amrex::ReduceData< - amrex::ParticleReal, amrex::ParticleReal, amrex::ParticleReal, - amrex::ParticleReal, amrex::ParticleReal, amrex::ParticleReal, - amrex::ParticleReal> - >( - pc, - [=] AMREX_GPU_DEVICE(const ConstPTDType& ptd, const int i) noexcept - { - amrex::ParticleReal const x = ptd.rdata(0)[i]; - amrex::ParticleReal const y = AMREX_SPACEDIM >= 2 ? ptd.rdata(1)[i] : 0.0; - amrex::ParticleReal const z = AMREX_SPACEDIM >= 3 ? ptd.rdata(2)[i] : 0.0; - - amrex::ParticleReal const w = ptd.rdata(AMREX_SPACEDIM)[i]; - - return amrex::makeTuple(x, x*x, y, y*y, z, z*z, w); - }, - reduce_ops - ); - amrex::ignore_unused(r); - - // Reduce for SoA Particle Struct - /* - using PTDType = typename T_PC::ParticleTileType::ConstParticleTileDataType; - amrex::ReduceOps reduce_ops; - auto r = amrex::ParticleReduce> ( - pc, [=] AMREX_GPU_DEVICE (const PTDType& ptd, const int i) noexcept - -> amrex::GpuTuple - { - - const amrex::Real a = ptd.rdata(1)[i]; - const amrex::Real b = ptd.rdata(2)[i]; - const int c = ptd.idata(1)[i]; - return {a, b, c}; - }, reduce_ops); - - AMREX_ALWAYS_ASSERT(amrex::get<0>(r) == amrex::Real(std::pow(256, AMREX_SPACEDIM))); - AMREX_ALWAYS_ASSERT(amrex::get<1>(r) == 2.0); - AMREX_ALWAYS_ASSERT(amrex::get<2>(r) == 1); - */ } - - - -int main(int argc, char* argv[]) +int main (int argc, char* argv[]) { amrex::Initialize(argc,argv); { - addParticles< ParticleContainerPureSoA<4, 2> > (); + addParticles(); } amrex::Finalize(); } From 588e0c626960b06eba10d76c97f584365c7e2fda Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Wed, 25 Sep 2024 12:28:27 -0700 Subject: [PATCH 05/10] remove unused type alias --- Tests/Particles/NamedSoAComponents/main.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/Tests/Particles/NamedSoAComponents/main.cpp b/Tests/Particles/NamedSoAComponents/main.cpp index bf9fbad0749..558fea47ff3 100644 --- a/Tests/Particles/NamedSoAComponents/main.cpp +++ b/Tests/Particles/NamedSoAComponents/main.cpp @@ -84,12 +84,9 @@ void addParticles () int const NArrayReal = pc.NArrayReal; int const NArrayInt = pc.NArrayInt; - using ParticleType = typename PC::ParticleType; - using ParticleTileDataType = typename PC::ParticleTileType::ParticleTileDataType; const int add_num_particles = 5; - auto& ptile1 = pc.DefineAndReturnParticleTile(0, 0, 0); ptile1.resize(add_num_particles); From 8abc2a4ca0141c594b419728fda7da6e9147b51b Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Wed, 25 Sep 2024 12:45:13 -0700 Subject: [PATCH 06/10] clang-tidy --- Tests/Particles/NamedSoAComponents/main.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Tests/Particles/NamedSoAComponents/main.cpp b/Tests/Particles/NamedSoAComponents/main.cpp index 558fea47ff3..ee5ebc5562c 100644 --- a/Tests/Particles/NamedSoAComponents/main.cpp +++ b/Tests/Particles/NamedSoAComponents/main.cpp @@ -82,8 +82,8 @@ void addParticles () } amrex::Print() << "\n"; - int const NArrayReal = pc.NArrayReal; - int const NArrayInt = pc.NArrayInt; + int const NArrayReal = PC::NArrayReal; + int const NArrayInt = PC::NArrayInt; using ParticleType = typename PC::ParticleType; const int add_num_particles = 5; @@ -97,7 +97,7 @@ void addParticles () } ptile1.getParticleTileData().rdata(AMREX_SPACEDIM)[i] = 1.2; // w - ptile1.push_back_int(0, ParticleType::NextID()); + ptile1.push_back_int(0, int(ParticleType::NextID())); ptile1.push_back_int(1, amrex::ParallelDescriptor::MyProc()); } @@ -105,10 +105,10 @@ void addParticles () using MyParIter = ParIter_impl; for (MyParIter pti(pc, lev); pti.isValid(); ++pti) { auto& soa = pti.GetStructOfArrays(); - auto xp = soa.GetRealData("x").data(); - auto yp = soa.GetRealData("y").data(); - auto zp = soa.GetRealData("z").data(); - auto wp = soa.GetRealData("w").data(); + auto *xp = soa.GetRealData("x").data(); + auto *yp = soa.GetRealData("y").data(); + auto *zp = soa.GetRealData("z").data(); + auto *wp = soa.GetRealData("w").data(); const int np = pti.numParticles(); ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) From 78d7db0fcec02a48b9f4711959f7277b3b677454 Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Wed, 25 Sep 2024 13:07:59 -0700 Subject: [PATCH 07/10] fix 1D and 2D --- Tests/Particles/NamedSoAComponents/main.cpp | 26 ++++++++++++--------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/Tests/Particles/NamedSoAComponents/main.cpp b/Tests/Particles/NamedSoAComponents/main.cpp index ee5ebc5562c..12655c441be 100644 --- a/Tests/Particles/NamedSoAComponents/main.cpp +++ b/Tests/Particles/NamedSoAComponents/main.cpp @@ -13,7 +13,7 @@ using namespace amrex; void addParticles () { - using PC = ParticleContainerPureSoA<4, 2>; + using PC = ParticleContainerPureSoA; int is_per[AMREX_SPACEDIM]; for (int & d : is_per) { d = 1; @@ -68,7 +68,7 @@ void addParticles () amrex::Print() << "\n"; amrex::Print() << "Reset compile-time SoA names \n"; - pc.SetSoACompileTimeNames({"x", "y", "z", "w"}, {"i1", "i2"}); + pc.SetSoACompileTimeNames({AMREX_D_DECL("x", "y", "z"), "w"}, {"i1", "i2"}); amrex::Print() << "New Real SoA component names are: "; for (auto& n : pc.GetRealSoANames()) { @@ -105,20 +105,24 @@ void addParticles () using MyParIter = ParIter_impl; for (MyParIter pti(pc, lev); pti.isValid(); ++pti) { auto& soa = pti.GetStructOfArrays(); - auto *xp = soa.GetRealData("x").data(); - auto *yp = soa.GetRealData("y").data(); - auto *zp = soa.GetRealData("z").data(); + AMREX_D_TERM( + auto *xp = soa.GetRealData("x").data();, + auto *yp = soa.GetRealData("y").data();, + auto *zp = soa.GetRealData("z").data(); + ); auto *wp = soa.GetRealData("w").data(); const int np = pti.numParticles(); ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) { - AMREX_ALWAYS_ASSERT_WITH_MESSAGE(xp[ip] == 12_prt, - "pos attribute expected to be 12"); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE(yp[ip] == 12_prt, - "pos attribute expected to be 12"); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE(zp[ip] == 12_prt, - "pos attribute expected to be 12"); + AMREX_D_TERM( + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(xp[ip] == 12_prt, + "pos attribute expected to be 12");, + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(yp[ip] == 12_prt, + "pos attribute expected to be 12");, + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(zp[ip] == 12_prt, + "pos attribute expected to be 12"); + ); AMREX_ALWAYS_ASSERT_WITH_MESSAGE(wp[ip] == 1.2_prt, "pos attribute expected to be 1.2"); }); From 7676c4d9e038b5fd06b9baec664568e184271221 Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Wed, 2 Oct 2024 11:51:48 -0700 Subject: [PATCH 08/10] use int instead of bool in AddRealComp --- Src/Particle/AMReX_ParticleContainer.H | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Src/Particle/AMReX_ParticleContainer.H b/Src/Particle/AMReX_ParticleContainer.H index 646c52a1411..3280acbdd71 100644 --- a/Src/Particle/AMReX_ParticleContainer.H +++ b/Src/Particle/AMReX_ParticleContainer.H @@ -1249,7 +1249,7 @@ public: Long superParticleSize() const { return superparticle_size; } - void AddRealComp (std::string const & name, bool communicate=true) + void AddRealComp (std::string const & name, int communicate=1) { m_soa_rdata_names.push_back(name); @@ -1272,13 +1272,13 @@ public: } } - void AddRealComp (bool communicate=true) + void AddRealComp (int communicate=1) { std::string default_name = "runtime_real" + std::to_string(m_num_runtime_real); AddRealComp(default_name, communicate); } - void AddIntComp (std::string const & name, bool communicate=true) + void AddIntComp (std::string const & name, int communicate=1) { m_soa_idata_names.push_back(name); @@ -1301,7 +1301,7 @@ public: } } - void AddIntComp (bool communicate=true) + void AddIntComp (int communicate=1) { std::string default_name = "runtime_int" + std::to_string(m_num_runtime_int); AddIntComp(default_name, communicate); From e7a88e16d2d23cbe84233017cc4da3b64437c02b Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Wed, 2 Oct 2024 15:01:06 -0700 Subject: [PATCH 09/10] refactor default names --- Src/Particle/AMReX_ParticleContainer.H | 6 +- Src/Particle/AMReX_ParticleContainerI.H | 17 +----- Src/Particle/AMReX_ParticleIO.H | 68 +++++++-------------- Src/Particle/AMReX_ParticleUtil.H | 20 ++++++ Tests/Particles/NamedSoAComponents/main.cpp | 6 +- 5 files changed, 50 insertions(+), 67 deletions(-) diff --git a/Src/Particle/AMReX_ParticleContainer.H b/Src/Particle/AMReX_ParticleContainer.H index 3280acbdd71..4b347d283b8 100644 --- a/Src/Particle/AMReX_ParticleContainer.H +++ b/Src/Particle/AMReX_ParticleContainer.H @@ -1274,8 +1274,7 @@ public: void AddRealComp (int communicate=1) { - std::string default_name = "runtime_real" + std::to_string(m_num_runtime_real); - AddRealComp(default_name, communicate); + AddRealComp(getDefaultCompNameReal(NArrayReal+m_num_runtime_real), communicate); } void AddIntComp (std::string const & name, int communicate=1) @@ -1303,8 +1302,7 @@ public: void AddIntComp (int communicate=1) { - std::string default_name = "runtime_int" + std::to_string(m_num_runtime_int); - AddIntComp(default_name, communicate); + AddIntComp(getDefaultCompNameInt(NArrayInt+m_num_runtime_int), communicate); } int NumRuntimeRealComps () const { return m_num_runtime_real; } diff --git a/Src/Particle/AMReX_ParticleContainerI.H b/Src/Particle/AMReX_ParticleContainerI.H index 6cd7cb6159c..718405f1e66 100644 --- a/Src/Particle/AMReX_ParticleContainerI.H +++ b/Src/Particle/AMReX_ParticleContainerI.H @@ -65,24 +65,13 @@ ParticleContainer_impl(i)); } for (int i=0; i(i)); } initialized = true; diff --git a/Src/Particle/AMReX_ParticleIO.H b/Src/Particle/AMReX_ParticleIO.H index 01ab0ded869..d10ab21b530 100644 --- a/Src/Particle/AMReX_ParticleIO.H +++ b/Src/Particle/AMReX_ParticleIO.H @@ -51,16 +51,14 @@ ParticleContainer_impl write_real_comp; Vector tmp_real_comp_names; - int nrc = ParticleType::is_soa_particle ? NStructReal + NumRealComps() - AMREX_SPACEDIM : NStructReal + NumRealComps(); - for (int i = 0; i < nrc; ++i ) + int first_rcomp = ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0; + for (int i = first_rcomp; i < NStructReal + NumRealComps(); ++i ) { write_real_comp.push_back(1); if (real_comp_names.empty()) { - std::stringstream ss; - ss << "real_comp" << i; - tmp_real_comp_names.push_back(ss.str()); + tmp_real_comp_names.push_back(getDefaultCompNameReal(i)); } else { @@ -75,9 +73,7 @@ ParticleContainer_impl(i)); } else { @@ -98,14 +94,12 @@ ParticleContainer_impl write_real_comp; Vector real_comp_names; - int nrc = ParticleType::is_soa_particle ? NStructReal + NumRealComps() - AMREX_SPACEDIM : NStructReal + NumRealComps(); - for (int i = 0; i < nrc; ++i ) + int first_rcomp = ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0; + for (int i = first_rcomp; i < NStructReal + NumRealComps(); ++i ) { write_real_comp.push_back(1); - std::stringstream ss; - ss << "real_comp" << i; - real_comp_names.push_back(ss.str()); + real_comp_names.push_back(getDefaultCompNameReal(i)); } Vector write_int_comp; @@ -113,9 +107,7 @@ ParticleContainer_impl(i)); } WriteBinaryParticleData(dir, name, write_real_comp, write_int_comp, @@ -182,9 +174,7 @@ ParticleContainer_impl int_comp_names; for (int i = 0; i < NStructInt + NumIntComps(); ++i ) { - std::stringstream ss; - ss << "int_comp" << i; - int_comp_names.push_back(ss.str()); + int_comp_names.push_back(getDefaultCompNameInt(i)); } WriteBinaryParticleData(dir, name, @@ -211,20 +201,16 @@ ParticleContainer_impl real_comp_names; - int nrc = ParticleType::is_soa_particle ? NStructReal + NumRealComps() - AMREX_SPACEDIM : NStructReal + NumRealComps(); - for (int i = 0; i < nrc; ++i ) + int first_rcomp = ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0; + for (int i = first_rcomp; i < NStructReal + NumRealComps(); ++i ) { - std::stringstream ss; - ss << "real_comp" << i; - real_comp_names.push_back(ss.str()); + real_comp_names.push_back(getDefaultCompNameReal(i)); } Vector int_comp_names; for (int i = 0; i < NStructInt + NumIntComps(); ++i ) { - std::stringstream ss; - ss << "int_comp" << i; - int_comp_names.push_back(ss.str()); + int_comp_names.push_back(getDefaultCompNameInt(i)); } WriteBinaryParticleData(dir, name, write_real_comp, write_int_comp, @@ -259,14 +245,12 @@ ParticleContainer_impl write_real_comp; Vector real_comp_names; - int nrc = ParticleType::is_soa_particle ? NStructReal + NumRealComps() - AMREX_SPACEDIM : NStructReal + NumRealComps(); - for (int i = 0; i < nrc; ++i ) + int first_rcomp = ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0; + for (int i = first_rcomp; i < NStructReal + NumRealComps(); ++i ) { write_real_comp.push_back(1); - std::stringstream ss; - ss << "real_comp" << i; - real_comp_names.push_back(ss.str()); + real_comp_names.push_back(getDefaultCompNameReal(i)); } Vector write_int_comp; @@ -274,9 +258,7 @@ ParticleContainer_impl(i)); } WriteBinaryParticleData(dir, name, write_real_comp, write_int_comp, @@ -345,9 +327,7 @@ ParticleContainer_impl int_comp_names; for (int i = 0; i < NStructInt + NumIntComps(); ++i ) { - std::stringstream ss; - ss << "int_comp" << i; - int_comp_names.push_back(ss.str()); + int_comp_names.push_back(getDefaultCompNameInt(i)); } WriteBinaryParticleData(dir, name, @@ -374,20 +354,16 @@ ParticleContainer_impl real_comp_names; - int nrc = ParticleType::is_soa_particle ? NStructReal + NumRealComps() - AMREX_SPACEDIM : NStructReal + NumRealComps(); - for (int i = 0; i < nrc; ++i ) + int first_rcomp = ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0; + for (int i = first_rcomp; i < NStructReal + NumRealComps(); ++i ) { - std::stringstream ss; - ss << "real_comp" << i; - real_comp_names.push_back(ss.str()); + real_comp_names.push_back(getDefaultCompNameReal(i)); } Vector int_comp_names; for (int i = 0; i < NStructInt + NumIntComps(); ++i ) { - std::stringstream ss; - ss << "int_comp" << i; - int_comp_names.push_back(ss.str()); + int_comp_names.push_back(getDefaultCompNameInt(i)); } WriteBinaryParticleData(dir, name, write_real_comp, write_int_comp, diff --git a/Src/Particle/AMReX_ParticleUtil.H b/Src/Particle/AMReX_ParticleUtil.H index b09f1d3583f..f6f2506ec7f 100644 --- a/Src/Particle/AMReX_ParticleUtil.H +++ b/Src/Particle/AMReX_ParticleUtil.H @@ -883,6 +883,26 @@ void PermutationForDeposition (Gpu::DeviceVector& perm, index_type n }); } +template +std::string getDefaultCompNameReal (const int i) { + int first_r_name = 0; + if constexpr (P::is_soa_particle) { + if (i < AMREX_SPACEDIM) { + constexpr int x_in_ascii = 120; + std::string const name{char(x_in_ascii+i)}; + return name; + } + first_r_name = AMREX_SPACEDIM; + } + std::string const name{("real_comp" + std::to_string(i-first_r_name))}; + return name; +} + +template +std::string getDefaultCompNameInt (const int i) { + std::string const name{("int_comp" + std::to_string(i))}; + return name; +} #ifdef AMREX_USE_HDF5_ASYNC void async_vol_es_wait_particle(); diff --git a/Tests/Particles/NamedSoAComponents/main.cpp b/Tests/Particles/NamedSoAComponents/main.cpp index 12655c441be..39b6a70ff52 100644 --- a/Tests/Particles/NamedSoAComponents/main.cpp +++ b/Tests/Particles/NamedSoAComponents/main.cpp @@ -51,9 +51,9 @@ void addParticles () amrex::Print() << "\n"; amrex::Print() << "Adding runtime comps. \n"; - pc.AddRealComp("runtime_rcomp0"); - pc.AddRealComp(); // without name - should be runtime_rcomp1 - pc.AddIntComp(); // without name - should be runtime_icomp0 + pc.AddRealComp("real_comp1"); + pc.AddRealComp(); // without name - should be real_comp2 + pc.AddIntComp(); // without name - should be int_comp0 amrex::Print() << "New Real SoA component names are: "; for (auto& n : pc.GetRealSoANames()) { From dbd9262e7a6ebfe102acdcef558f47a2544aeabf Mon Sep 17 00:00:00 2001 From: Andrew Myers Date: Wed, 2 Oct 2024 15:20:43 -0700 Subject: [PATCH 10/10] fix indexing bug --- Src/Particle/AMReX_ParticleIO.H | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Src/Particle/AMReX_ParticleIO.H b/Src/Particle/AMReX_ParticleIO.H index d10ab21b530..601b7417d18 100644 --- a/Src/Particle/AMReX_ParticleIO.H +++ b/Src/Particle/AMReX_ParticleIO.H @@ -62,7 +62,7 @@ ParticleContainer_impl