Start EB (#504)

This is still work in progress. You can build EB and create
EBFArrayBoxFactory objects from python.

---------

Signed-off-by: Axel Huebl <axel.huebl@plasma.ninja>
Co-authored-by: Axel Huebl <axel.huebl@plasma.ninja>

Author: WeiqunZhang

docs/source/usage/api.rst

@@ -280,3 +280,19 @@ This is for the legacy, AoS + SoA particle containers only:
 .. autoclass:: amrex.space3d.Particle_2_1
    :members:
    :undoc-members:
+
+.. _usage-api-eb:
+
+Embedded Boundaries
+-------------------
+
+Embedded boundary (EB) support in pyAMReX is still minimal. To build pyAMReX with
+EB support, you need to add ``-DAMReX_EB=ON`` to CMake build options.
+
+.. autofunction:: amrex.space3d.EB2_Build
+
+.. autoclass:: amrex.space3d.EBFArrayBoxFactory
+   :members:
+   :undoc-members:
+
+.. autofunction:: amrex.space3d.makeEBFabFactory

pyAMReXConfig.cmake.in

@@ -26,6 +26,7 @@ set(pyAMReX_CUDA                      @AMReX_CUDA@)
 set(pyAMReX_SYCL                      @AMReX_SYCL@)
 set(pyAMReX_HIP                       @AMReX_HIP@)
 set(pyAMReX_GPU_BACKEND               @AMReX_GPU_BACKEND@)
+set(pyAMReX_EB                        @AMReX_EB@)
 
 # define central pyAMReX::pyAMReX_${D}d targets
 include("${CMAKE_CURRENT_LIST_DIR}/pyAMReXTargets.cmake")

setup.py

@@ -97,6 +97,7 @@ def build_extension(self, ext):
             "-DAMReX_MPI:BOOL=" + AMReX_MPI,
             "-DAMReX_PRECISION=" + AMReX_PRECISION,
             "-DAMReX_PARTICLES_PRECISION=" + AMReX_PARTICLES_PRECISION,
+            "-DAMReX_EB:BOOL=" + AMReX_EB,
             "-DpyAMReX_CCACHE=" + PYAMREX_CCACHE,
             "-DpyAMReX_IPO=" + PYAMREX_IPO,
             ## dependency control (developers & package managers)
@@ -176,6 +177,7 @@ def build_extension(self, ext):
 AMReX_MPI = os.environ.get("AMREX_MPI", "OFF")
 AMReX_PRECISION = os.environ.get("AMREX_PRECISION", "DOUBLE")
 AMReX_PARTICLES_PRECISION = os.environ.get("AMREX_PARTICLES_PRECISION", "DOUBLE")
+AMReX_EB = os.environ.get("AMREX_EB", "OFF")
 #   single value or as a list 1;2;3
 AMReX_SPACEDIM = os.environ.get("AMREX_SPACEDIM", "1;2;3")
 BUILD_SHARED_LIBS = os.environ.get("AMREX_BUILD_SHARED_LIBS", "OFF")

src/CMakeLists.txt

@@ -2,7 +2,7 @@
 add_subdirectory(AmrCore)
 add_subdirectory(Base)
 #add_subdirectory(Boundary)
-#add_subdirectory(EB)
+add_subdirectory(EB)
 #add_subdirectory(Extern)
 #add_subdirectory(LinearSolvers)
 add_subdirectory(Particle)

src/EB/CMakeLists.txt

@@ -0,0 +1,13 @@
+if (NOT AMReX_EB)
+    return()
+endif()
+
+foreach(D IN LISTS AMReX_SPACEDIM)
+    if (D GREATER 1)
+        target_sources(pyAMReX_${D}d
+          PRIVATE
+            EB.cpp
+            EBFabFactory.cpp
+        )
+    endif()
+endforeach()

src/EB/EB.cpp

@@ -0,0 +1,34 @@
+/* Copyright 2022 The AMReX Community
+ *
+ * Authors: Weiqun Zhang, Axel Huebl
+ * License: BSD-3-Clause-LBNL
+ */
+#include "pyAMReX.H"
+
+#include <AMReX_EB2.H>
+
+
+void init_EBFabFactory (py::module& m);
+
+void init_EB (py::module& m)
+{
+    using namespace amrex;
+
+    m.def(
+        "EB2_Build",
+        [] (Geometry const& geom, int required_coarsening_level, int max_coarsening_level,
+            int ngrow, bool build_coarse_level_by_coarsening, bool extend_domain_face,
+            int num_coarsen_opt)
+        {
+            EB2::Build(geom, required_coarsening_level, max_coarsening_level, ngrow,
+                       build_coarse_level_by_coarsening, extend_domain_face, num_coarsen_opt);
+        },
+        py::arg("geom"), py::arg("required_coarsening_level"), py::arg("max_coarsening_level"),
+        py::arg("ngrow") = 4, py::arg("build_coarse_level_by_coarsening") = true,
+        py::arg("extend_domain_face") = EB2::ExtendDomainFace(),
+        py::arg("num_coarsen_opt") = EB2::NumCoarsenOpt(),
+        "EB generation"
+    );
+
+    init_EBFabFactory(m);
+}

src/EB/EBFabFactory.cpp

@@ -0,0 +1,40 @@
+/* Copyright 2022 The AMReX Community
+ *
+ * Authors: Weiqun Zhang, Axel Huebl
+ * License: BSD-3-Clause-LBNL
+ */
+#include "pyAMReX.H"
+
+#include <AMReX_EBFabFactory.H>
+#include <AMReX_MultiFab.H>
+
+
+void init_EBFabFactory (py::module& m)
+{
+    using namespace amrex;
+
+    py::class_<EBFArrayBoxFactory, FabFactory<FArrayBox>>(m, "EBFArrayBoxFactory")
+        .def("getVolFrac", &EBFArrayBoxFactory::getVolFrac,
+            py::return_value_policy::reference_internal,
+            "Return volume faction MultiFab");
+
+    py::native_enum<EBSupport>(m, "EBSupport", "enum.Enum")
+        .value("basic", EBSupport::basic)
+        .value("volume", EBSupport::volume)
+        .value("full", EBSupport::full)
+        .export_values()
+        .finalize()
+    ;
+
+    m.def(
+        "makeEBFabFactory",
+        [] (Geometry const& geom, BoxArray const& ba, DistributionMapping const& dm,
+            Vector<int> const& ngrow, EBSupport support)
+        {
+            return makeEBFabFactory(geom, ba, dm, ngrow, support);
+        },
+        py::arg("geom"), py::arg("ba"), py::arg("dm"), py::arg("ngrow"),
+        py::arg("support"),
+        "Make EBFArrayBoxFactory for given Geometry, BoxArray and DistributionMapping"
+    );
+}

src/pyAMReX.cpp

@@ -48,6 +48,9 @@ void init_Utility(py::module &);
 void init_Vector(py::module &);
 void init_Version(py::module &);
 void init_VisMF(py::module &);
+#ifdef AMREX_USE_EB
+void init_EB(py::module &);
+#endif
 
 #if AMREX_SPACEDIM == 1
 PYBIND11_MODULE(amrex_1d_pybind, m) {
@@ -139,6 +142,10 @@ PYBIND11_MODULE(amrex_3d_pybind, m) {
     init_Version(m);
     init_VisMF(m);
 
+#ifdef AMREX_USE_EB
+    init_EB(m);
+#endif
+
     // authors
     m.attr("__author__") =
         "Axel Huebl, Ryan T. Sandberg, Shreyas Ananthan, David P. Grote, "

tests/test_eb.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+
+import numpy
+import pytest
+
+import amrex.space3d as amr
+
+
+@pytest.mark.skipif(not amr.Config.have_eb, reason="Requires -DAMReX_EB=ON")
+def test_makeEBFabFactory():
+    n_cell = 64
+    max_grid_size = 16
+
+    # Build Geometry
+    domain = amr.Box(
+        amr.IntVect(0, 0, 0), amr.IntVect(n_cell - 1, n_cell - 1, n_cell - 1)
+    )
+    real_box = amr.RealBox([0.0, 0.0, 0.0], [1.0, 1.0, 1.0])
+    coord = 0  # Cartesian
+    is_per = [1, 1, 1]  # is periodic?
+    geom = amr.Geometry(domain, real_box, coord, is_per)
+
+    # EB parameters
+    pp = amr.ParmParse("eb2")
+    pp.add("geom_type", "sphere")
+    pp.addarr("sphere_center", [0.5, 0.5, 0.5])
+    rsphere = 0.25
+    pp.add("sphere_radius", rsphere)
+    pp.add("sphere_has_fluid_inside", 1)
+
+    # EB generation
+    eb_requried_level = 0
+    eb_max_level = 2
+    amr.EB2_Build(geom, eb_requried_level, eb_max_level)
+
+    # Build BoxArray
+    ba = amr.BoxArray(domain)
+    ba.max_size(max_grid_size)
+
+    # Build DistributionMapping
+    dm = amr.DistributionMapping(ba)
+
+    # Make EB Factory
+    ng = amr.Vector_int([1, 1, 1])
+    factory = amr.makeEBFabFactory(geom, ba, dm, ng, amr.EBSupport.full)
+
+    # Get EB data
+    vfrac = factory.getVolFrac()
+
+    dx = geom.data().CellSize()
+    total_vol = vfrac.sum() * dx[0] * dx[1] * dx[2]
+    sphere_vol = 4.0 / 3.0 * numpy.pi * rsphere**3
+    assert abs(sphere_vol - total_vol) / sphere_vol < 2.0e-3