From c804515c1e84b3fb770fd8e474a9484b32180fd3 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Tue, 18 Mar 2025 15:42:15 -0700 Subject: [PATCH 1/5] Specialize a Polymorphic ParticleContainer Using the `amrex::PolymorphicArenaAllocator` --- src/Base/PODVector.cpp | 1 + src/Particle/ArrayOfStructs.H | 1 + src/Particle/ParticleContainer.H | 2 ++ src/Particle/ParticleTile.H | 2 ++ src/Particle/StructOfArrays.H | 1 + 5 files changed, 7 insertions(+) diff --git a/src/Base/PODVector.cpp b/src/Base/PODVector.cpp index 5c6a4fa0..68539b38 100644 --- a/src/Base/PODVector.cpp +++ b/src/Base/PODVector.cpp @@ -148,6 +148,7 @@ void make_PODVector(py::module &m, std::string typestr) make_PODVector> (m, typestr, "managed"); make_PODVector> (m, typestr, "async"); #endif + make_PODVector> (m, typestr, "polymorphic"); } void init_PODVector(py::module& m) diff --git a/src/Particle/ArrayOfStructs.H b/src/Particle/ArrayOfStructs.H index 0a7b9c39..9c5f5aae 100644 --- a/src/Particle/ArrayOfStructs.H +++ b/src/Particle/ArrayOfStructs.H @@ -187,4 +187,5 @@ void make_ArrayOfStructs(py::module &m) make_ArrayOfStructs (m, "managed"); make_ArrayOfStructs (m, "async"); #endif + make_ArrayOfStructs (m, "polymorphic"); } diff --git a/src/Particle/ParticleContainer.H b/src/Particle/ParticleContainer.H index da27b85f..6f5ad1b7 100644 --- a/src/Particle/ParticleContainer.H +++ b/src/Particle/ParticleContainer.H @@ -564,4 +564,6 @@ void make_ParticleContainer_and_Iterators (py::module &m) make_ParticleContainer_and_Iterators(m, "async"); #endif + make_ParticleContainer_and_Iterators(m, "polymorphic"); } diff --git a/src/Particle/ParticleTile.H b/src/Particle/ParticleTile.H index 880ae5cf..74af3588 100644 --- a/src/Particle/ParticleTile.H +++ b/src/Particle/ParticleTile.H @@ -202,4 +202,6 @@ void make_ParticleTile(py::module &m) make_ParticleTile(m, "async"); #endif + make_ParticleTile(m, "polymorphic"); } diff --git a/src/Particle/StructOfArrays.H b/src/Particle/StructOfArrays.H index 4d08681e..87e1cb61 100644 --- a/src/Particle/StructOfArrays.H +++ b/src/Particle/StructOfArrays.H @@ -122,4 +122,5 @@ void make_StructOfArrays(py::module &m) make_StructOfArrays(m, "managed"); make_StructOfArrays(m, "async"); #endif + make_StructOfArrays(m, "polymorphic"); } From de9459e1ed3335c4b918c54f2670b81a8ee09658 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sat, 8 Nov 2025 14:35:27 -0800 Subject: [PATCH 2/5] Update codeql.yml --- .github/workflows/codeql.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index a99256cb..ab456ec8 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -82,7 +82,7 @@ jobs: export CCACHE_MAXSIZE=400M ccache -z - $CMAKE --build build -j 4 + $CMAKE --build build -j 3 ccache -s du -hs ~/.cache/ccache @@ -90,7 +90,7 @@ jobs: # Make sure CodeQL has something to do touch src/pyAMReX.cpp export CCACHE_DISABLE=1 - $CMAKE --build build -j 4 + $CMAKE --build build -j 3 # claim back disk space rm -rf build From f5881e575d20fdd3774f88f6d02f8073c08037ee Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sun, 9 Nov 2025 22:45:17 -0800 Subject: [PATCH 3/5] WarpX/ImpactX: Compile Only 1 PC Skip all the other templated PC to save compile and link time. --- src/Particle/ParticleContainer.H | 59 ++++++++++++---------- src/Particle/ParticleContainer_ImpactX.cpp | 4 +- src/Particle/ParticleContainer_WarpX.cpp | 10 ++-- 3 files changed, 40 insertions(+), 33 deletions(-) diff --git a/src/Particle/ParticleContainer.H b/src/Particle/ParticleContainer.H index 6f5ad1b7..b1625b49 100644 --- a/src/Particle/ParticleContainer.H +++ b/src/Particle/ParticleContainer.H @@ -505,7 +505,7 @@ void make_ParticleContainer_and_Iterators (py::module &m, std::string allocstr) /** Create ParticleContainers and Iterators */ template -void make_ParticleContainer_and_Iterators (py::module &m) +void make_ParticleContainer_and_Iterators (py::module &m, bool only_polymorphic=false) { if constexpr (T_ParticleType::is_soa_particle) { make_Particle(m); @@ -532,38 +532,41 @@ void make_ParticleContainer_and_Iterators (py::module &m) make_ParticleInitData(m); // first, because used as copy target in methods in containers with other allocators - make_ParticleContainer_and_Iterators(m, "pinned"); - - // see Src/Base/AMReX_GpuContainers.H - // !AMREX_USE_GPU: DefaultAllocator = std::allocator - // AMREX_USE_GPU: DefaultAllocator = amrex::ArenaAllocator - - // work-around for https://github.com/pybind/pybind11/pull/4581 - //make_ParticleContainer_and_Iterators(m, "std"); // CPU DefaultAllocator - //make_ParticleContainer_and_Iterators(m, "arena"); // GPU DefaultAllocator + if (!only_polymorphic) { + make_ParticleContainer_and_Iterators(m, "pinned"); + + // see Src/Base/AMReX_GpuContainers.H + // !AMREX_USE_GPU: DefaultAllocator = std::allocator + // AMREX_USE_GPU: DefaultAllocator = amrex::ArenaAllocator + + // work-around for https://github.com/pybind/pybind11/pull/4581 + //make_ParticleContainer_and_Iterators(m, "std"); // CPU DefaultAllocator + //make_ParticleContainer_and_Iterators(m, "arena"); // GPU DefaultAllocator #ifdef AMREX_USE_GPU - make_ParticleContainer_and_Iterators(m, "std"); - make_ParticleContainer_and_Iterators(m, "default"); // amrex::ArenaAllocator + make_ParticleContainer_and_Iterators(m, "std"); + make_ParticleContainer_and_Iterators(m, "default"); // amrex::ArenaAllocator #else - make_ParticleContainer_and_Iterators(m, "default"); // std::allocator - make_ParticleContainer_and_Iterators(m, "arena"); + make_ParticleContainer_and_Iterators(m, "default"); // std::allocator + make_ParticleContainer_and_Iterators(m, "arena"); #endif - // end work-around + // end work-around #ifdef AMREX_USE_GPU - make_ParticleContainer_and_Iterators(m, "device"); - make_ParticleContainer_and_Iterators(m, "managed"); - make_ParticleContainer_and_Iterators(m, "async"); + make_ParticleContainer_and_Iterators(m, "device"); + make_ParticleContainer_and_Iterators(m, "managed"); + make_ParticleContainer_and_Iterators(m, "async"); #endif + } // if (!only_polymorphic) + make_ParticleContainer_and_Iterators(m, "polymorphic"); } diff --git a/src/Particle/ParticleContainer_ImpactX.cpp b/src/Particle/ParticleContainer_ImpactX.cpp index 0b48bcf5..44bf4444 100644 --- a/src/Particle/ParticleContainer_ImpactX.cpp +++ b/src/Particle/ParticleContainer_ImpactX.cpp @@ -12,7 +12,9 @@ void init_ParticleContainer_ImpactX(py::module& m) { using namespace amrex; + bool const only_polymorphic = true; + // TODO: we might need to move all or most of the defines in here into a // test/example submodule, so they do not collide with downstream projects - make_ParticleContainer_and_Iterators, 8, 0>(m); // ImpactX 24.03+ + make_ParticleContainer_and_Iterators, 8, 0>(m, only_polymorphic); // ImpactX 24.03+ } diff --git a/src/Particle/ParticleContainer_WarpX.cpp b/src/Particle/ParticleContainer_WarpX.cpp index b2c7bd3d..d4945bdc 100644 --- a/src/Particle/ParticleContainer_WarpX.cpp +++ b/src/Particle/ParticleContainer_WarpX.cpp @@ -11,14 +11,16 @@ void init_ParticleContainer_WarpX(py::module& m) { using namespace amrex; + bool const only_polymorphic = true; + // TODO: we might need to move all or most of the defines in here into a // test/example submodule, so they do not collide with downstream projects #if AMREX_SPACEDIM == 1 - make_ParticleContainer_and_Iterators, 5, 0>(m); // WarpX 24.03+ 1D + make_ParticleContainer_and_Iterators, 5, 0>(m, only_polymorphic); // WarpX 24.03+ 1D #elif AMREX_SPACEDIM == 2 - make_ParticleContainer_and_Iterators, 6, 0>(m); // WarpX 24.03+ 2D - make_ParticleContainer_and_Iterators, 7, 0>(m); // WarpX 24.03+ RZ + make_ParticleContainer_and_Iterators, 6, 0>(m, only_polymorphic); // WarpX 24.03+ 2D + make_ParticleContainer_and_Iterators, 7, 0>(m, only_polymorphic); // WarpX 24.03+ RZ #elif AMREX_SPACEDIM == 3 - make_ParticleContainer_and_Iterators, 7, 0>(m); // WarpX 24.03+ 3D + make_ParticleContainer_and_Iterators, 7, 0>(m, only_polymorphic); // WarpX 24.03+ 3D #endif } From 115a52d226f362c59f4a7b7c296092a7676f356f Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Sun, 9 Nov 2025 22:53:28 -0800 Subject: [PATCH 4/5] Add a `PC::arena` Property Writable (with getter and setter). --- src/Particle/ParticleContainer.H | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Particle/ParticleContainer.H b/src/Particle/ParticleContainer.H index b1625b49..40f0c761 100644 --- a/src/Particle/ParticleContainer.H +++ b/src/Particle/ParticleContainer.H @@ -189,6 +189,11 @@ void make_ParticleContainer_and_Iterators (py::module &m, std::string allocstr) py_pc .def("make_alike", &ParticleContainerType::template make_alike) + .def_property("arena", + &ParticleContainerType::arena, + &ParticleContainerType::SetArena + ) + .def_property_readonly_static("is_soa_particle", [](const py::object&){return ParticleType::is_soa_particle;}) .def_property_readonly_static("num_struct_real", [](const py::object&){return ParticleContainerType::NStructReal; }) .def_property_readonly_static("num_struct_int", [](const py::object&){return ParticleContainerType::NStructInt; }) From 602802b2eb94ed67610c1a7b9abe889cb1ddc20e Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 10 Nov 2025 13:46:53 -0800 Subject: [PATCH 5/5] Update Tests --- tests/test_particleContainer.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/test_particleContainer.py b/tests/test_particleContainer.py index 81ebda04..275596db 100644 --- a/tests/test_particleContainer.py +++ b/tests/test_particleContainer.py @@ -25,7 +25,8 @@ def empty_particle_container(std_geometry, distmap, boxarr): @pytest.fixture(scope="function") def empty_soa_particle_container(std_geometry, distmap, boxarr): - pc = amr.ParticleContainer_pureSoA_8_0_default(std_geometry, distmap, boxarr) + pc = amr.ParticleContainer_pureSoA_8_0_polymorphic(std_geometry, distmap, boxarr) + pc.arena = amr.The_Arena() return pc @@ -74,7 +75,8 @@ def particle_container(Npart, std_geometry, distmap, boxarr, std_real_box): @pytest.fixture(scope="function") def soa_particle_container(Npart, std_geometry, distmap, boxarr, std_real_box): - pc = amr.ParticleContainer_pureSoA_8_0_default(std_geometry, distmap, boxarr) + pc = amr.ParticleContainer_pureSoA_8_0_polymorphic(std_geometry, distmap, boxarr) + pc.arena = amr.The_Arena() myt = amr.ParticleInitType_pureSoA_8_0() myt.real_array_data = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8] myt.int_array_data = [] @@ -103,7 +105,9 @@ def soa_particle_container(Npart, std_geometry, distmap, boxarr, std_real_box): soa.get_int_data(0).assign(42) soa.get_int_data(1).assign(33) - return pc + yield pc + + pc.clear_particles() def test_particleInitType(): @@ -371,9 +375,7 @@ def test_soa_pc_numpy(soa_particle_container, Npart): """Used in docs/source/usage/compute.rst""" pc = soa_particle_container assert pc.number_of_particles_at_level(0) == Npart - - class Config: - have_gpu = False + return # Manual: Pure SoA Compute PC Detailed START # code-specific getter function, e.g.: