From 6a61340afccdfdaca989fd78609fe45ba345350d Mon Sep 17 00:00:00 2001
From: Michael Zingale <michael.zingale@stonybrook.edu>
Date: Thu, 14 Sep 2023 11:50:25 -0400
Subject: [PATCH] allow the random number generator on GPUs

---
 Exec/Make.Castro                     | 4 ++++
 Exec/science/xrb_layered/GNUmakefile | 1 +
 Source/driver/Castro.cpp             | 2 +-
 Source/driver/Castro_advance.cpp     | 5 +++++
 Source/driver/Castro_io.cpp          | 4 ++++
 5 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/Exec/Make.Castro b/Exec/Make.Castro
index 2db7d6aa36..e31c452842 100644
--- a/Exec/Make.Castro
+++ b/Exec/Make.Castro
@@ -166,6 +166,10 @@ ifeq ($(USE_CXX_MODEL_PARSER), TRUE)
   DEFINES += -DCXX_MODEL_PARSER -DNPTS_MODEL=$(MAX_NPTS_MODEL) -DNUM_MODELS=$(NUM_MODELS)
 endif
 
+ifeq ($(USE_RNG_STATE_INIT), TRUE)
+  DEFINES += -DRNG_STATE_INIT
+endif
+
 # add / define any special physics we need
 
 ifeq ($(USE_MHD), TRUE)
diff --git a/Exec/science/xrb_layered/GNUmakefile b/Exec/science/xrb_layered/GNUmakefile
index 516b853f83..f029cb400a 100644
--- a/Exec/science/xrb_layered/GNUmakefile
+++ b/Exec/science/xrb_layered/GNUmakefile
@@ -12,6 +12,7 @@ USE_GRAV         = TRUE
 USE_REACT        = TRUE
 
 USE_CXX_MODEL_PARSER = TRUE
+USE_RNG_STATE_INIT = TRUE
 
 CASTRO_HOME = ../../..
 
diff --git a/Source/driver/Castro.cpp b/Source/driver/Castro.cpp
index 7ccd6a33a5..3e0ad948ea 100644
--- a/Source/driver/Castro.cpp
+++ b/Source/driver/Castro.cpp
@@ -1105,7 +1105,7 @@ Castro::initData ()
           auto s = S_new[mfi].array();
           auto geomdata = geom.data();
 
-#ifdef USE_RNG_STATE_INIT
+#ifdef RNG_STATE_INIT
           amrex::ParallelForRNG(box,
           [=] AMREX_GPU_HOST_DEVICE (int i, int j, int k, amrex::RandomEngine const& engine) noexcept
           {
diff --git a/Source/driver/Castro_advance.cpp b/Source/driver/Castro_advance.cpp
index aa3e068748..a7190c2fd7 100644
--- a/Source/driver/Castro_advance.cpp
+++ b/Source/driver/Castro_advance.cpp
@@ -428,6 +428,9 @@ Castro::initialize_advance(Real time, Real dt, int amr_iteration)
                 auto s = S_old[mfi].array();
                 auto geomdata = geom.data();
 
+#ifdef RNG_STATE_INIT
+                amrex::Error("drive initial convection not yet supported for random initialization");
+#else
                 amrex::ParallelFor(box,
                 [=] AMREX_GPU_HOST_DEVICE (int i, int j, int k)
                 {
@@ -449,6 +452,8 @@ Castro::initialize_advance(Real time, Real dt, int amr_iteration)
                         (vx_orig * vx_orig + vy_orig * vy_orig + vz_orig * vz_orig);
 
                 });
+#endif
+
             }
 
         }
diff --git a/Source/driver/Castro_io.cpp b/Source/driver/Castro_io.cpp
index e3bb00a5d2..33fde99403 100644
--- a/Source/driver/Castro_io.cpp
+++ b/Source/driver/Castro_io.cpp
@@ -291,6 +291,9 @@ Castro::restart (Amr&     papa,
                auto s = S_new[mfi].array();
                auto geomdata = geom.data();
 
+#ifdef RNG_STATE_INIT
+               amrex::Error("Error: random initialization not yet supported with grown factor");
+#else
                amrex::ParallelFor(bx,
                [=] AMREX_GPU_HOST_DEVICE (int i, int j, int k)
                {
@@ -298,6 +301,7 @@ Castro::restart (Amr&     papa,
                    // by a problem setup (defaults to an empty routine).
                    problem_initialize_state_data(i, j, k, s, geomdata);
                });
+#endif
            }
        }
     }